blob: 2843813290d8bc625b5c625ffbe133a5a69f7e0c [file] [log] [blame]
Eric Anholt673a3942008-07-30 12:06:12 -07001/*
Daniel Vetterbe6a0372015-03-18 10:46:04 +01002 * Copyright © 2008-2015 Intel Corporation
Eric Anholt673a3942008-07-30 12:06:12 -07003 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
David Howells760285e2012-10-02 18:01:07 +010028#include <drm/drmP.h>
David Herrmann0de23972013-07-24 21:07:52 +020029#include <drm/drm_vma_manager.h>
David Howells760285e2012-10-02 18:01:07 +010030#include <drm/i915_drm.h>
Eric Anholt673a3942008-07-30 12:06:12 -070031#include "i915_drv.h"
Yu Zhangeb822892015-02-10 19:05:49 +080032#include "i915_vgpu.h"
Chris Wilson1c5d22f2009-08-25 11:15:50 +010033#include "i915_trace.h"
Jesse Barnes652c3932009-08-17 13:31:43 -070034#include "intel_drv.h"
Hugh Dickins5949eac2011-06-27 16:18:18 -070035#include <linux/shmem_fs.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090036#include <linux/slab.h>
Eric Anholt673a3942008-07-30 12:06:12 -070037#include <linux/swap.h>
Jesse Barnes79e53942008-11-07 14:24:08 -080038#include <linux/pci.h>
Daniel Vetter1286ff72012-05-10 15:25:09 +020039#include <linux/dma-buf.h>
Eric Anholt673a3942008-07-30 12:06:12 -070040
Chris Wilsonb4716182015-04-27 13:41:17 +010041#define RQ_BUG_ON(expr)
42
Chris Wilson05394f32010-11-08 19:18:58 +000043static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
Daniel Vettere62b59e2015-01-21 14:53:48 +010044static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
Chris Wilsonc8725f32014-03-17 12:21:55 +000045static void
Chris Wilsonb4716182015-04-27 13:41:17 +010046i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
47static void
48i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
Chris Wilson61050802012-04-17 15:31:31 +010049
Chris Wilsonc76ce032013-08-08 14:41:03 +010050static bool cpu_cache_is_coherent(struct drm_device *dev,
51 enum i915_cache_level level)
52{
53 return HAS_LLC(dev) || level != I915_CACHE_NONE;
54}
55
Chris Wilson2c225692013-08-09 12:26:45 +010056static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
57{
58 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
59 return true;
60
61 return obj->pin_display;
62}
63
Chris Wilson73aa8082010-09-30 11:46:12 +010064/* some bookkeeping */
65static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
66 size_t size)
67{
Daniel Vetterc20e8352013-07-24 22:40:23 +020068 spin_lock(&dev_priv->mm.object_stat_lock);
Chris Wilson73aa8082010-09-30 11:46:12 +010069 dev_priv->mm.object_count++;
70 dev_priv->mm.object_memory += size;
Daniel Vetterc20e8352013-07-24 22:40:23 +020071 spin_unlock(&dev_priv->mm.object_stat_lock);
Chris Wilson73aa8082010-09-30 11:46:12 +010072}
73
74static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
75 size_t size)
76{
Daniel Vetterc20e8352013-07-24 22:40:23 +020077 spin_lock(&dev_priv->mm.object_stat_lock);
Chris Wilson73aa8082010-09-30 11:46:12 +010078 dev_priv->mm.object_count--;
79 dev_priv->mm.object_memory -= size;
Daniel Vetterc20e8352013-07-24 22:40:23 +020080 spin_unlock(&dev_priv->mm.object_stat_lock);
Chris Wilson73aa8082010-09-30 11:46:12 +010081}
82
Chris Wilson21dd3732011-01-26 15:55:56 +000083static int
Daniel Vetter33196de2012-11-14 17:14:05 +010084i915_gem_wait_for_error(struct i915_gpu_error *error)
Chris Wilson30dbf0c2010-09-25 10:19:17 +010085{
Chris Wilson30dbf0c2010-09-25 10:19:17 +010086 int ret;
87
Daniel Vetter7abb6902013-05-24 21:29:32 +020088#define EXIT_COND (!i915_reset_in_progress(error) || \
89 i915_terminally_wedged(error))
Daniel Vetter1f83fee2012-11-15 17:17:22 +010090 if (EXIT_COND)
Chris Wilson30dbf0c2010-09-25 10:19:17 +010091 return 0;
92
Daniel Vetter0a6759c2012-07-04 22:18:41 +020093 /*
94 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
95 * userspace. If it takes that long something really bad is going on and
96 * we should simply try to bail out and fail as gracefully as possible.
97 */
Daniel Vetter1f83fee2012-11-15 17:17:22 +010098 ret = wait_event_interruptible_timeout(error->reset_queue,
99 EXIT_COND,
100 10*HZ);
Daniel Vetter0a6759c2012-07-04 22:18:41 +0200101 if (ret == 0) {
102 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
103 return -EIO;
104 } else if (ret < 0) {
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100105 return ret;
Daniel Vetter0a6759c2012-07-04 22:18:41 +0200106 }
Daniel Vetter1f83fee2012-11-15 17:17:22 +0100107#undef EXIT_COND
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100108
Chris Wilson21dd3732011-01-26 15:55:56 +0000109 return 0;
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100110}
111
Chris Wilson54cf91d2010-11-25 18:00:26 +0000112int i915_mutex_lock_interruptible(struct drm_device *dev)
Chris Wilson76c1dec2010-09-25 11:22:51 +0100113{
Daniel Vetter33196de2012-11-14 17:14:05 +0100114 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson76c1dec2010-09-25 11:22:51 +0100115 int ret;
116
Daniel Vetter33196de2012-11-14 17:14:05 +0100117 ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
Chris Wilson76c1dec2010-09-25 11:22:51 +0100118 if (ret)
119 return ret;
120
121 ret = mutex_lock_interruptible(&dev->struct_mutex);
122 if (ret)
123 return ret;
124
Chris Wilson23bc5982010-09-29 16:10:57 +0100125 WARN_ON(i915_verify_lists(dev));
Chris Wilson76c1dec2010-09-25 11:22:51 +0100126 return 0;
127}
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100128
Eric Anholt673a3942008-07-30 12:06:12 -0700129int
Eric Anholt5a125c32008-10-22 21:40:13 -0700130i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +0000131 struct drm_file *file)
Eric Anholt5a125c32008-10-22 21:40:13 -0700132{
Joonas Lahtinen72e96d62016-03-30 16:57:10 +0300133 struct drm_i915_private *dev_priv = to_i915(dev);
Joonas Lahtinen62106b42016-03-18 10:42:57 +0200134 struct i915_ggtt *ggtt = &dev_priv->ggtt;
Joonas Lahtinen72e96d62016-03-30 16:57:10 +0300135 struct drm_i915_gem_get_aperture *args = data;
Tvrtko Ursulinca1543b2015-07-01 11:51:10 +0100136 struct i915_vma *vma;
Chris Wilson6299f992010-11-24 12:23:44 +0000137 size_t pinned;
Eric Anholt5a125c32008-10-22 21:40:13 -0700138
Chris Wilson6299f992010-11-24 12:23:44 +0000139 pinned = 0;
Chris Wilson73aa8082010-09-30 11:46:12 +0100140 mutex_lock(&dev->struct_mutex);
Chris Wilson1c7f4bc2016-02-26 11:03:19 +0000141 list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
Tvrtko Ursulinca1543b2015-07-01 11:51:10 +0100142 if (vma->pin_count)
143 pinned += vma->node.size;
Chris Wilson1c7f4bc2016-02-26 11:03:19 +0000144 list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
Tvrtko Ursulinca1543b2015-07-01 11:51:10 +0100145 if (vma->pin_count)
146 pinned += vma->node.size;
Chris Wilson73aa8082010-09-30 11:46:12 +0100147 mutex_unlock(&dev->struct_mutex);
Eric Anholt5a125c32008-10-22 21:40:13 -0700148
Joonas Lahtinen72e96d62016-03-30 16:57:10 +0300149 args->aper_size = ggtt->base.total;
Akshay Joshi0206e352011-08-16 15:34:10 -0400150 args->aper_available_size = args->aper_size - pinned;
Chris Wilson6299f992010-11-24 12:23:44 +0000151
Eric Anholt5a125c32008-10-22 21:40:13 -0700152 return 0;
153}
154
Chris Wilson6a2c4232014-11-04 04:51:40 -0800155static int
156i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
Chris Wilson00731152014-05-21 12:42:56 +0100157{
Chris Wilson6a2c4232014-11-04 04:51:40 -0800158 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
159 char *vaddr = obj->phys_handle->vaddr;
160 struct sg_table *st;
161 struct scatterlist *sg;
162 int i;
Chris Wilson00731152014-05-21 12:42:56 +0100163
Chris Wilson6a2c4232014-11-04 04:51:40 -0800164 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
165 return -EINVAL;
Chris Wilson00731152014-05-21 12:42:56 +0100166
Chris Wilson6a2c4232014-11-04 04:51:40 -0800167 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
168 struct page *page;
169 char *src;
170
171 page = shmem_read_mapping_page(mapping, i);
172 if (IS_ERR(page))
173 return PTR_ERR(page);
174
175 src = kmap_atomic(page);
176 memcpy(vaddr, src, PAGE_SIZE);
177 drm_clflush_virt_range(vaddr, PAGE_SIZE);
178 kunmap_atomic(src);
179
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300180 put_page(page);
Chris Wilson6a2c4232014-11-04 04:51:40 -0800181 vaddr += PAGE_SIZE;
182 }
183
184 i915_gem_chipset_flush(obj->base.dev);
185
186 st = kmalloc(sizeof(*st), GFP_KERNEL);
187 if (st == NULL)
188 return -ENOMEM;
189
190 if (sg_alloc_table(st, 1, GFP_KERNEL)) {
191 kfree(st);
192 return -ENOMEM;
193 }
194
195 sg = st->sgl;
196 sg->offset = 0;
197 sg->length = obj->base.size;
198
199 sg_dma_address(sg) = obj->phys_handle->busaddr;
200 sg_dma_len(sg) = obj->base.size;
201
202 obj->pages = st;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800203 return 0;
204}
205
206static void
207i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
208{
209 int ret;
210
211 BUG_ON(obj->madv == __I915_MADV_PURGED);
212
213 ret = i915_gem_object_set_to_cpu_domain(obj, true);
214 if (ret) {
215 /* In the event of a disaster, abandon all caches and
216 * hope for the best.
217 */
218 WARN_ON(ret != -EIO);
219 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
220 }
221
222 if (obj->madv == I915_MADV_DONTNEED)
223 obj->dirty = 0;
224
225 if (obj->dirty) {
Chris Wilson00731152014-05-21 12:42:56 +0100226 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800227 char *vaddr = obj->phys_handle->vaddr;
Chris Wilson00731152014-05-21 12:42:56 +0100228 int i;
229
230 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
Chris Wilson6a2c4232014-11-04 04:51:40 -0800231 struct page *page;
232 char *dst;
Chris Wilson00731152014-05-21 12:42:56 +0100233
Chris Wilson6a2c4232014-11-04 04:51:40 -0800234 page = shmem_read_mapping_page(mapping, i);
235 if (IS_ERR(page))
236 continue;
237
238 dst = kmap_atomic(page);
239 drm_clflush_virt_range(vaddr, PAGE_SIZE);
240 memcpy(dst, vaddr, PAGE_SIZE);
241 kunmap_atomic(dst);
242
243 set_page_dirty(page);
244 if (obj->madv == I915_MADV_WILLNEED)
Chris Wilson00731152014-05-21 12:42:56 +0100245 mark_page_accessed(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300246 put_page(page);
Chris Wilson00731152014-05-21 12:42:56 +0100247 vaddr += PAGE_SIZE;
248 }
Chris Wilson6a2c4232014-11-04 04:51:40 -0800249 obj->dirty = 0;
Chris Wilson00731152014-05-21 12:42:56 +0100250 }
251
Chris Wilson6a2c4232014-11-04 04:51:40 -0800252 sg_free_table(obj->pages);
253 kfree(obj->pages);
Chris Wilson6a2c4232014-11-04 04:51:40 -0800254}
255
256static void
257i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
258{
259 drm_pci_free(obj->base.dev, obj->phys_handle);
260}
261
262static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
263 .get_pages = i915_gem_object_get_pages_phys,
264 .put_pages = i915_gem_object_put_pages_phys,
265 .release = i915_gem_object_release_phys,
266};
267
268static int
269drop_pages(struct drm_i915_gem_object *obj)
270{
271 struct i915_vma *vma, *next;
272 int ret;
273
274 drm_gem_object_reference(&obj->base);
Chris Wilson1c7f4bc2016-02-26 11:03:19 +0000275 list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link)
Chris Wilson6a2c4232014-11-04 04:51:40 -0800276 if (i915_vma_unbind(vma))
277 break;
278
279 ret = i915_gem_object_put_pages(obj);
280 drm_gem_object_unreference(&obj->base);
281
282 return ret;
Chris Wilson00731152014-05-21 12:42:56 +0100283}
284
285int
286i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
287 int align)
288{
289 drm_dma_handle_t *phys;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800290 int ret;
Chris Wilson00731152014-05-21 12:42:56 +0100291
292 if (obj->phys_handle) {
293 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
294 return -EBUSY;
295
296 return 0;
297 }
298
299 if (obj->madv != I915_MADV_WILLNEED)
300 return -EFAULT;
301
302 if (obj->base.filp == NULL)
303 return -EINVAL;
304
Chris Wilson6a2c4232014-11-04 04:51:40 -0800305 ret = drop_pages(obj);
306 if (ret)
307 return ret;
308
Chris Wilson00731152014-05-21 12:42:56 +0100309 /* create a new object */
310 phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
311 if (!phys)
312 return -ENOMEM;
313
Chris Wilson00731152014-05-21 12:42:56 +0100314 obj->phys_handle = phys;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800315 obj->ops = &i915_gem_phys_ops;
316
317 return i915_gem_object_get_pages(obj);
Chris Wilson00731152014-05-21 12:42:56 +0100318}
319
320static int
321i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
322 struct drm_i915_gem_pwrite *args,
323 struct drm_file *file_priv)
324{
325 struct drm_device *dev = obj->base.dev;
326 void *vaddr = obj->phys_handle->vaddr + args->offset;
327 char __user *user_data = to_user_ptr(args->data_ptr);
Paulo Zanoni063e4e62015-02-13 17:23:45 -0200328 int ret = 0;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800329
330 /* We manually control the domain here and pretend that it
331 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
332 */
333 ret = i915_gem_object_wait_rendering(obj, false);
334 if (ret)
335 return ret;
Chris Wilson00731152014-05-21 12:42:56 +0100336
Rodrigo Vivi77a0d1c2015-06-18 11:43:24 -0700337 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
Chris Wilson00731152014-05-21 12:42:56 +0100338 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
339 unsigned long unwritten;
340
341 /* The physical object once assigned is fixed for the lifetime
342 * of the obj, so we can safely drop the lock and continue
343 * to access vaddr.
344 */
345 mutex_unlock(&dev->struct_mutex);
346 unwritten = copy_from_user(vaddr, user_data, args->size);
347 mutex_lock(&dev->struct_mutex);
Paulo Zanoni063e4e62015-02-13 17:23:45 -0200348 if (unwritten) {
349 ret = -EFAULT;
350 goto out;
351 }
Chris Wilson00731152014-05-21 12:42:56 +0100352 }
353
Chris Wilson6a2c4232014-11-04 04:51:40 -0800354 drm_clflush_virt_range(vaddr, args->size);
Chris Wilson00731152014-05-21 12:42:56 +0100355 i915_gem_chipset_flush(dev);
Paulo Zanoni063e4e62015-02-13 17:23:45 -0200356
357out:
Rodrigo Vivide152b62015-07-07 16:28:51 -0700358 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
Paulo Zanoni063e4e62015-02-13 17:23:45 -0200359 return ret;
Chris Wilson00731152014-05-21 12:42:56 +0100360}
361
Chris Wilson42dcedd2012-11-15 11:32:30 +0000362void *i915_gem_object_alloc(struct drm_device *dev)
363{
364 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilsonefab6d82015-04-07 16:20:57 +0100365 return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
Chris Wilson42dcedd2012-11-15 11:32:30 +0000366}
367
368void i915_gem_object_free(struct drm_i915_gem_object *obj)
369{
370 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
Chris Wilsonefab6d82015-04-07 16:20:57 +0100371 kmem_cache_free(dev_priv->objects, obj);
Chris Wilson42dcedd2012-11-15 11:32:30 +0000372}
373
Dave Airlieff72145b2011-02-07 12:16:14 +1000374static int
375i915_gem_create(struct drm_file *file,
376 struct drm_device *dev,
377 uint64_t size,
378 uint32_t *handle_p)
Eric Anholt673a3942008-07-30 12:06:12 -0700379{
Chris Wilson05394f32010-11-08 19:18:58 +0000380 struct drm_i915_gem_object *obj;
Pekka Paalanena1a2d1d2009-08-23 12:40:55 +0300381 int ret;
382 u32 handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700383
Dave Airlieff72145b2011-02-07 12:16:14 +1000384 size = roundup(size, PAGE_SIZE);
Chris Wilson8ffc0242011-09-14 14:14:28 +0200385 if (size == 0)
386 return -EINVAL;
Eric Anholt673a3942008-07-30 12:06:12 -0700387
388 /* Allocate the new object */
Dave Airlieff72145b2011-02-07 12:16:14 +1000389 obj = i915_gem_alloc_object(dev, size);
Eric Anholt673a3942008-07-30 12:06:12 -0700390 if (obj == NULL)
391 return -ENOMEM;
392
Chris Wilson05394f32010-11-08 19:18:58 +0000393 ret = drm_gem_handle_create(file, &obj->base, &handle);
Chris Wilson202f2fe2010-10-14 13:20:40 +0100394 /* drop reference from allocate - handle holds it now */
Daniel Vetterd861e332013-07-24 23:25:03 +0200395 drm_gem_object_unreference_unlocked(&obj->base);
396 if (ret)
397 return ret;
Chris Wilson202f2fe2010-10-14 13:20:40 +0100398
Dave Airlieff72145b2011-02-07 12:16:14 +1000399 *handle_p = handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700400 return 0;
401}
402
Dave Airlieff72145b2011-02-07 12:16:14 +1000403int
404i915_gem_dumb_create(struct drm_file *file,
405 struct drm_device *dev,
406 struct drm_mode_create_dumb *args)
407{
408 /* have to work out size/pitch and return them */
Paulo Zanonide45eaf2013-10-18 18:48:24 -0300409 args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
Dave Airlieff72145b2011-02-07 12:16:14 +1000410 args->size = args->pitch * args->height;
411 return i915_gem_create(file, dev,
Dave Airlieda6b51d2014-12-24 13:11:17 +1000412 args->size, &args->handle);
Dave Airlieff72145b2011-02-07 12:16:14 +1000413}
414
Dave Airlieff72145b2011-02-07 12:16:14 +1000415/**
416 * Creates a new mm object and returns a handle to it.
417 */
418int
419i915_gem_create_ioctl(struct drm_device *dev, void *data,
420 struct drm_file *file)
421{
422 struct drm_i915_gem_create *args = data;
Daniel Vetter63ed2cb2012-04-23 16:50:50 +0200423
Dave Airlieff72145b2011-02-07 12:16:14 +1000424 return i915_gem_create(file, dev,
Dave Airlieda6b51d2014-12-24 13:11:17 +1000425 args->size, &args->handle);
Dave Airlieff72145b2011-02-07 12:16:14 +1000426}
427
Daniel Vetter8c599672011-12-14 13:57:31 +0100428static inline int
Daniel Vetter8461d222011-12-14 13:57:32 +0100429__copy_to_user_swizzled(char __user *cpu_vaddr,
430 const char *gpu_vaddr, int gpu_offset,
431 int length)
432{
433 int ret, cpu_offset = 0;
434
435 while (length > 0) {
436 int cacheline_end = ALIGN(gpu_offset + 1, 64);
437 int this_length = min(cacheline_end - gpu_offset, length);
438 int swizzled_gpu_offset = gpu_offset ^ 64;
439
440 ret = __copy_to_user(cpu_vaddr + cpu_offset,
441 gpu_vaddr + swizzled_gpu_offset,
442 this_length);
443 if (ret)
444 return ret + length;
445
446 cpu_offset += this_length;
447 gpu_offset += this_length;
448 length -= this_length;
449 }
450
451 return 0;
452}
453
454static inline int
Ben Widawsky4f0c7cf2012-04-16 14:07:47 -0700455__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
456 const char __user *cpu_vaddr,
Daniel Vetter8c599672011-12-14 13:57:31 +0100457 int length)
458{
459 int ret, cpu_offset = 0;
460
461 while (length > 0) {
462 int cacheline_end = ALIGN(gpu_offset + 1, 64);
463 int this_length = min(cacheline_end - gpu_offset, length);
464 int swizzled_gpu_offset = gpu_offset ^ 64;
465
466 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
467 cpu_vaddr + cpu_offset,
468 this_length);
469 if (ret)
470 return ret + length;
471
472 cpu_offset += this_length;
473 gpu_offset += this_length;
474 length -= this_length;
475 }
476
477 return 0;
478}
479
Brad Volkin4c914c02014-02-18 10:15:45 -0800480/*
481 * Pins the specified object's pages and synchronizes the object with
482 * GPU accesses. Sets needs_clflush to non-zero if the caller should
483 * flush the object from the CPU cache.
484 */
485int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
486 int *needs_clflush)
487{
488 int ret;
489
490 *needs_clflush = 0;
491
Ben Widawsky1db6e2e2016-02-09 11:44:12 -0800492 if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
Brad Volkin4c914c02014-02-18 10:15:45 -0800493 return -EINVAL;
494
495 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
496 /* If we're not in the cpu read domain, set ourself into the gtt
497 * read domain and manually flush cachelines (if required). This
498 * optimizes for the case when the gpu will dirty the data
499 * anyway again before the next pread happens. */
500 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
501 obj->cache_level);
502 ret = i915_gem_object_wait_rendering(obj, true);
503 if (ret)
504 return ret;
505 }
506
507 ret = i915_gem_object_get_pages(obj);
508 if (ret)
509 return ret;
510
511 i915_gem_object_pin_pages(obj);
512
513 return ret;
514}
515
Daniel Vetterd174bd62012-03-25 19:47:40 +0200516/* Per-page copy function for the shmem pread fastpath.
517 * Flushes invalid cachelines before reading the target if
518 * needs_clflush is set. */
Eric Anholteb014592009-03-10 11:44:52 -0700519static int
Daniel Vetterd174bd62012-03-25 19:47:40 +0200520shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
521 char __user *user_data,
522 bool page_do_bit17_swizzling, bool needs_clflush)
523{
524 char *vaddr;
525 int ret;
526
Daniel Vettere7e58eb2012-03-25 19:47:43 +0200527 if (unlikely(page_do_bit17_swizzling))
Daniel Vetterd174bd62012-03-25 19:47:40 +0200528 return -EINVAL;
529
530 vaddr = kmap_atomic(page);
531 if (needs_clflush)
532 drm_clflush_virt_range(vaddr + shmem_page_offset,
533 page_length);
534 ret = __copy_to_user_inatomic(user_data,
535 vaddr + shmem_page_offset,
536 page_length);
537 kunmap_atomic(vaddr);
538
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100539 return ret ? -EFAULT : 0;
Daniel Vetterd174bd62012-03-25 19:47:40 +0200540}
541
Daniel Vetter23c18c72012-03-25 19:47:42 +0200542static void
543shmem_clflush_swizzled_range(char *addr, unsigned long length,
544 bool swizzled)
545{
Daniel Vettere7e58eb2012-03-25 19:47:43 +0200546 if (unlikely(swizzled)) {
Daniel Vetter23c18c72012-03-25 19:47:42 +0200547 unsigned long start = (unsigned long) addr;
548 unsigned long end = (unsigned long) addr + length;
549
550 /* For swizzling simply ensure that we always flush both
551 * channels. Lame, but simple and it works. Swizzled
552 * pwrite/pread is far from a hotpath - current userspace
553 * doesn't use it at all. */
554 start = round_down(start, 128);
555 end = round_up(end, 128);
556
557 drm_clflush_virt_range((void *)start, end - start);
558 } else {
559 drm_clflush_virt_range(addr, length);
560 }
561
562}
563
Daniel Vetterd174bd62012-03-25 19:47:40 +0200564/* Only difference to the fast-path function is that this can handle bit17
565 * and uses non-atomic copy and kmap functions. */
566static int
567shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
568 char __user *user_data,
569 bool page_do_bit17_swizzling, bool needs_clflush)
570{
571 char *vaddr;
572 int ret;
573
574 vaddr = kmap(page);
575 if (needs_clflush)
Daniel Vetter23c18c72012-03-25 19:47:42 +0200576 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
577 page_length,
578 page_do_bit17_swizzling);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200579
580 if (page_do_bit17_swizzling)
581 ret = __copy_to_user_swizzled(user_data,
582 vaddr, shmem_page_offset,
583 page_length);
584 else
585 ret = __copy_to_user(user_data,
586 vaddr + shmem_page_offset,
587 page_length);
588 kunmap(page);
589
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100590 return ret ? - EFAULT : 0;
Daniel Vetterd174bd62012-03-25 19:47:40 +0200591}
592
Eric Anholteb014592009-03-10 11:44:52 -0700593static int
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200594i915_gem_shmem_pread(struct drm_device *dev,
595 struct drm_i915_gem_object *obj,
596 struct drm_i915_gem_pread *args,
597 struct drm_file *file)
Eric Anholteb014592009-03-10 11:44:52 -0700598{
Daniel Vetter8461d222011-12-14 13:57:32 +0100599 char __user *user_data;
Eric Anholteb014592009-03-10 11:44:52 -0700600 ssize_t remain;
Daniel Vetter8461d222011-12-14 13:57:32 +0100601 loff_t offset;
Ben Widawskyeb2c0c82012-02-15 14:42:43 +0100602 int shmem_page_offset, page_length, ret = 0;
Daniel Vetter8461d222011-12-14 13:57:32 +0100603 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
Daniel Vetter96d79b52012-03-25 19:47:36 +0200604 int prefaulted = 0;
Daniel Vetter84897312012-03-25 19:47:31 +0200605 int needs_clflush = 0;
Imre Deak67d5a502013-02-18 19:28:02 +0200606 struct sg_page_iter sg_iter;
Eric Anholteb014592009-03-10 11:44:52 -0700607
Ville Syrjälä2bb46292013-02-22 16:12:51 +0200608 user_data = to_user_ptr(args->data_ptr);
Eric Anholteb014592009-03-10 11:44:52 -0700609 remain = args->size;
610
Daniel Vetter8461d222011-12-14 13:57:32 +0100611 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700612
Brad Volkin4c914c02014-02-18 10:15:45 -0800613 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100614 if (ret)
615 return ret;
616
Eric Anholteb014592009-03-10 11:44:52 -0700617 offset = args->offset;
Daniel Vetter8461d222011-12-14 13:57:32 +0100618
Imre Deak67d5a502013-02-18 19:28:02 +0200619 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
620 offset >> PAGE_SHIFT) {
Imre Deak2db76d72013-03-26 15:14:18 +0200621 struct page *page = sg_page_iter_page(&sg_iter);
Chris Wilson9da3da62012-06-01 15:20:22 +0100622
623 if (remain <= 0)
624 break;
625
Eric Anholteb014592009-03-10 11:44:52 -0700626 /* Operation in this page
627 *
Eric Anholteb014592009-03-10 11:44:52 -0700628 * shmem_page_offset = offset within page in shmem file
Eric Anholteb014592009-03-10 11:44:52 -0700629 * page_length = bytes to copy for this page
630 */
Chris Wilsonc8cbbb82011-05-12 22:17:11 +0100631 shmem_page_offset = offset_in_page(offset);
Eric Anholteb014592009-03-10 11:44:52 -0700632 page_length = remain;
633 if ((shmem_page_offset + page_length) > PAGE_SIZE)
634 page_length = PAGE_SIZE - shmem_page_offset;
Eric Anholteb014592009-03-10 11:44:52 -0700635
Daniel Vetter8461d222011-12-14 13:57:32 +0100636 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
637 (page_to_phys(page) & (1 << 17)) != 0;
638
Daniel Vetterd174bd62012-03-25 19:47:40 +0200639 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
640 user_data, page_do_bit17_swizzling,
641 needs_clflush);
642 if (ret == 0)
643 goto next_page;
Eric Anholteb014592009-03-10 11:44:52 -0700644
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200645 mutex_unlock(&dev->struct_mutex);
646
Jani Nikulad330a952014-01-21 11:24:25 +0200647 if (likely(!i915.prefault_disable) && !prefaulted) {
Daniel Vetterf56f8212012-03-25 19:47:41 +0200648 ret = fault_in_multipages_writeable(user_data, remain);
Daniel Vetter96d79b52012-03-25 19:47:36 +0200649 /* Userspace is tricking us, but we've already clobbered
650 * its pages with the prefault and promised to write the
651 * data up to the first fault. Hence ignore any errors
652 * and just continue. */
653 (void)ret;
654 prefaulted = 1;
655 }
656
Daniel Vetterd174bd62012-03-25 19:47:40 +0200657 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
658 user_data, page_do_bit17_swizzling,
659 needs_clflush);
Eric Anholteb014592009-03-10 11:44:52 -0700660
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200661 mutex_lock(&dev->struct_mutex);
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100662
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100663 if (ret)
Daniel Vetter8461d222011-12-14 13:57:32 +0100664 goto out;
Daniel Vetter8461d222011-12-14 13:57:32 +0100665
Chris Wilson17793c92014-03-07 08:30:36 +0000666next_page:
Eric Anholteb014592009-03-10 11:44:52 -0700667 remain -= page_length;
Daniel Vetter8461d222011-12-14 13:57:32 +0100668 user_data += page_length;
Eric Anholteb014592009-03-10 11:44:52 -0700669 offset += page_length;
670 }
671
Chris Wilson4f27b752010-10-14 15:26:45 +0100672out:
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100673 i915_gem_object_unpin_pages(obj);
674
Eric Anholteb014592009-03-10 11:44:52 -0700675 return ret;
676}
677
Eric Anholt673a3942008-07-30 12:06:12 -0700678/**
679 * Reads data from the object referenced by handle.
680 *
681 * On error, the contents of *data are undefined.
682 */
683int
684i915_gem_pread_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +0000685 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -0700686{
687 struct drm_i915_gem_pread *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +0000688 struct drm_i915_gem_object *obj;
Chris Wilson35b62a82010-09-26 20:23:38 +0100689 int ret = 0;
Eric Anholt673a3942008-07-30 12:06:12 -0700690
Chris Wilson51311d02010-11-17 09:10:42 +0000691 if (args->size == 0)
692 return 0;
693
694 if (!access_ok(VERIFY_WRITE,
Ville Syrjälä2bb46292013-02-22 16:12:51 +0200695 to_user_ptr(args->data_ptr),
Chris Wilson51311d02010-11-17 09:10:42 +0000696 args->size))
697 return -EFAULT;
698
Chris Wilson4f27b752010-10-14 15:26:45 +0100699 ret = i915_mutex_lock_interruptible(dev);
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100700 if (ret)
Chris Wilson4f27b752010-10-14 15:26:45 +0100701 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700702
Chris Wilson05394f32010-11-08 19:18:58 +0000703 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +0000704 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100705 ret = -ENOENT;
706 goto unlock;
Chris Wilson4f27b752010-10-14 15:26:45 +0100707 }
Eric Anholt673a3942008-07-30 12:06:12 -0700708
Chris Wilson7dcd2492010-09-26 20:21:44 +0100709 /* Bounds check source. */
Chris Wilson05394f32010-11-08 19:18:58 +0000710 if (args->offset > obj->base.size ||
711 args->size > obj->base.size - args->offset) {
Chris Wilsonce9d4192010-09-26 20:50:05 +0100712 ret = -EINVAL;
Chris Wilson35b62a82010-09-26 20:23:38 +0100713 goto out;
Chris Wilsonce9d4192010-09-26 20:50:05 +0100714 }
715
Daniel Vetter1286ff72012-05-10 15:25:09 +0200716 /* prime objects have no backing filp to GEM pread/pwrite
717 * pages from.
718 */
719 if (!obj->base.filp) {
720 ret = -EINVAL;
721 goto out;
722 }
723
Chris Wilsondb53a302011-02-03 11:57:46 +0000724 trace_i915_gem_object_pread(obj, args->offset, args->size);
725
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200726 ret = i915_gem_shmem_pread(dev, obj, args, file);
Eric Anholt673a3942008-07-30 12:06:12 -0700727
Chris Wilson35b62a82010-09-26 20:23:38 +0100728out:
Chris Wilson05394f32010-11-08 19:18:58 +0000729 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100730unlock:
Chris Wilson4f27b752010-10-14 15:26:45 +0100731 mutex_unlock(&dev->struct_mutex);
Eric Anholteb014592009-03-10 11:44:52 -0700732 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700733}
734
Keith Packard0839ccb2008-10-30 19:38:48 -0700735/* This is the fast write path which cannot handle
736 * page faults in the source data
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700737 */
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700738
Keith Packard0839ccb2008-10-30 19:38:48 -0700739static inline int
740fast_user_write(struct io_mapping *mapping,
741 loff_t page_base, int page_offset,
742 char __user *user_data,
743 int length)
744{
Ben Widawsky4f0c7cf2012-04-16 14:07:47 -0700745 void __iomem *vaddr_atomic;
746 void *vaddr;
Keith Packard0839ccb2008-10-30 19:38:48 -0700747 unsigned long unwritten;
748
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -0700749 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
Ben Widawsky4f0c7cf2012-04-16 14:07:47 -0700750 /* We can use the cpu mem copy function because this is X86. */
751 vaddr = (void __force*)vaddr_atomic + page_offset;
752 unwritten = __copy_from_user_inatomic_nocache(vaddr,
Keith Packard0839ccb2008-10-30 19:38:48 -0700753 user_data, length);
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -0700754 io_mapping_unmap_atomic(vaddr_atomic);
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100755 return unwritten;
Keith Packard0839ccb2008-10-30 19:38:48 -0700756}
757
Eric Anholt3de09aa2009-03-09 09:42:23 -0700758/**
759 * This is the fast pwrite path, where we copy the data directly from the
760 * user into the GTT, uncached.
761 */
Eric Anholt673a3942008-07-30 12:06:12 -0700762static int
Chris Wilson05394f32010-11-08 19:18:58 +0000763i915_gem_gtt_pwrite_fast(struct drm_device *dev,
764 struct drm_i915_gem_object *obj,
Eric Anholt3de09aa2009-03-09 09:42:23 -0700765 struct drm_i915_gem_pwrite *args,
Chris Wilson05394f32010-11-08 19:18:58 +0000766 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -0700767{
Joonas Lahtinen72e96d62016-03-30 16:57:10 +0300768 struct drm_i915_private *dev_priv = to_i915(dev);
769 struct i915_ggtt *ggtt = &dev_priv->ggtt;
Eric Anholt673a3942008-07-30 12:06:12 -0700770 ssize_t remain;
Keith Packard0839ccb2008-10-30 19:38:48 -0700771 loff_t offset, page_base;
Eric Anholt673a3942008-07-30 12:06:12 -0700772 char __user *user_data;
Daniel Vetter935aaa62012-03-25 19:47:35 +0200773 int page_offset, page_length, ret;
774
Daniel Vetter1ec9e262014-02-14 14:01:11 +0100775 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
Daniel Vetter935aaa62012-03-25 19:47:35 +0200776 if (ret)
777 goto out;
778
779 ret = i915_gem_object_set_to_gtt_domain(obj, true);
780 if (ret)
781 goto out_unpin;
782
783 ret = i915_gem_object_put_fence(obj);
784 if (ret)
785 goto out_unpin;
Eric Anholt673a3942008-07-30 12:06:12 -0700786
Ville Syrjälä2bb46292013-02-22 16:12:51 +0200787 user_data = to_user_ptr(args->data_ptr);
Eric Anholt673a3942008-07-30 12:06:12 -0700788 remain = args->size;
Eric Anholt673a3942008-07-30 12:06:12 -0700789
Ben Widawskyf343c5f2013-07-05 14:41:04 -0700790 offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700791
Rodrigo Vivi77a0d1c2015-06-18 11:43:24 -0700792 intel_fb_obj_invalidate(obj, ORIGIN_GTT);
Paulo Zanoni063e4e62015-02-13 17:23:45 -0200793
Eric Anholt673a3942008-07-30 12:06:12 -0700794 while (remain > 0) {
795 /* Operation in this page
796 *
Keith Packard0839ccb2008-10-30 19:38:48 -0700797 * page_base = page offset within aperture
798 * page_offset = offset within page
799 * page_length = bytes to copy for this page
Eric Anholt673a3942008-07-30 12:06:12 -0700800 */
Chris Wilsonc8cbbb82011-05-12 22:17:11 +0100801 page_base = offset & PAGE_MASK;
802 page_offset = offset_in_page(offset);
Keith Packard0839ccb2008-10-30 19:38:48 -0700803 page_length = remain;
804 if ((page_offset + remain) > PAGE_SIZE)
805 page_length = PAGE_SIZE - page_offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700806
Keith Packard0839ccb2008-10-30 19:38:48 -0700807 /* If we get a fault while copying data, then (presumably) our
Eric Anholt3de09aa2009-03-09 09:42:23 -0700808 * source page isn't available. Return the error and we'll
809 * retry in the slow path.
Keith Packard0839ccb2008-10-30 19:38:48 -0700810 */
Joonas Lahtinen72e96d62016-03-30 16:57:10 +0300811 if (fast_user_write(ggtt->mappable, page_base,
Daniel Vetter935aaa62012-03-25 19:47:35 +0200812 page_offset, user_data, page_length)) {
813 ret = -EFAULT;
Paulo Zanoni063e4e62015-02-13 17:23:45 -0200814 goto out_flush;
Daniel Vetter935aaa62012-03-25 19:47:35 +0200815 }
Eric Anholt673a3942008-07-30 12:06:12 -0700816
Keith Packard0839ccb2008-10-30 19:38:48 -0700817 remain -= page_length;
818 user_data += page_length;
819 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700820 }
Eric Anholt673a3942008-07-30 12:06:12 -0700821
Paulo Zanoni063e4e62015-02-13 17:23:45 -0200822out_flush:
Rodrigo Vivide152b62015-07-07 16:28:51 -0700823 intel_fb_obj_flush(obj, false, ORIGIN_GTT);
Daniel Vetter935aaa62012-03-25 19:47:35 +0200824out_unpin:
Ben Widawskyd7f46fc2013-12-06 14:10:55 -0800825 i915_gem_object_ggtt_unpin(obj);
Daniel Vetter935aaa62012-03-25 19:47:35 +0200826out:
Eric Anholt3de09aa2009-03-09 09:42:23 -0700827 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700828}
829
Daniel Vetterd174bd62012-03-25 19:47:40 +0200830/* Per-page copy function for the shmem pwrite fastpath.
831 * Flushes invalid cachelines before writing to the target if
832 * needs_clflush_before is set and flushes out any written cachelines after
833 * writing if needs_clflush is set. */
Eric Anholt673a3942008-07-30 12:06:12 -0700834static int
Daniel Vetterd174bd62012-03-25 19:47:40 +0200835shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
836 char __user *user_data,
837 bool page_do_bit17_swizzling,
838 bool needs_clflush_before,
839 bool needs_clflush_after)
Eric Anholt673a3942008-07-30 12:06:12 -0700840{
Daniel Vetterd174bd62012-03-25 19:47:40 +0200841 char *vaddr;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700842 int ret;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700843
Daniel Vettere7e58eb2012-03-25 19:47:43 +0200844 if (unlikely(page_do_bit17_swizzling))
Daniel Vetterd174bd62012-03-25 19:47:40 +0200845 return -EINVAL;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700846
Daniel Vetterd174bd62012-03-25 19:47:40 +0200847 vaddr = kmap_atomic(page);
848 if (needs_clflush_before)
849 drm_clflush_virt_range(vaddr + shmem_page_offset,
850 page_length);
Chris Wilsonc2831a92014-03-07 08:30:37 +0000851 ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
852 user_data, page_length);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200853 if (needs_clflush_after)
854 drm_clflush_virt_range(vaddr + shmem_page_offset,
855 page_length);
856 kunmap_atomic(vaddr);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700857
Chris Wilson755d2212012-09-04 21:02:55 +0100858 return ret ? -EFAULT : 0;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700859}
860
Daniel Vetterd174bd62012-03-25 19:47:40 +0200861/* Only difference to the fast-path function is that this can handle bit17
862 * and uses non-atomic copy and kmap functions. */
Eric Anholt3043c602008-10-02 12:24:47 -0700863static int
Daniel Vetterd174bd62012-03-25 19:47:40 +0200864shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
865 char __user *user_data,
866 bool page_do_bit17_swizzling,
867 bool needs_clflush_before,
868 bool needs_clflush_after)
Eric Anholt673a3942008-07-30 12:06:12 -0700869{
Daniel Vetterd174bd62012-03-25 19:47:40 +0200870 char *vaddr;
871 int ret;
Eric Anholt40123c12009-03-09 13:42:30 -0700872
Daniel Vetterd174bd62012-03-25 19:47:40 +0200873 vaddr = kmap(page);
Daniel Vettere7e58eb2012-03-25 19:47:43 +0200874 if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
Daniel Vetter23c18c72012-03-25 19:47:42 +0200875 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
876 page_length,
877 page_do_bit17_swizzling);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200878 if (page_do_bit17_swizzling)
879 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
Chris Wilsone5281cc2010-10-28 13:45:36 +0100880 user_data,
881 page_length);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200882 else
883 ret = __copy_from_user(vaddr + shmem_page_offset,
884 user_data,
885 page_length);
886 if (needs_clflush_after)
Daniel Vetter23c18c72012-03-25 19:47:42 +0200887 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
888 page_length,
889 page_do_bit17_swizzling);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200890 kunmap(page);
Chris Wilsone5281cc2010-10-28 13:45:36 +0100891
Chris Wilson755d2212012-09-04 21:02:55 +0100892 return ret ? -EFAULT : 0;
Eric Anholt40123c12009-03-09 13:42:30 -0700893}
894
Eric Anholt40123c12009-03-09 13:42:30 -0700895static int
Daniel Vettere244a442012-03-25 19:47:28 +0200896i915_gem_shmem_pwrite(struct drm_device *dev,
897 struct drm_i915_gem_object *obj,
898 struct drm_i915_gem_pwrite *args,
899 struct drm_file *file)
Eric Anholt40123c12009-03-09 13:42:30 -0700900{
Eric Anholt40123c12009-03-09 13:42:30 -0700901 ssize_t remain;
Daniel Vetter8c599672011-12-14 13:57:31 +0100902 loff_t offset;
903 char __user *user_data;
Ben Widawskyeb2c0c82012-02-15 14:42:43 +0100904 int shmem_page_offset, page_length, ret = 0;
Daniel Vetter8c599672011-12-14 13:57:31 +0100905 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
Daniel Vettere244a442012-03-25 19:47:28 +0200906 int hit_slowpath = 0;
Daniel Vetter58642882012-03-25 19:47:37 +0200907 int needs_clflush_after = 0;
908 int needs_clflush_before = 0;
Imre Deak67d5a502013-02-18 19:28:02 +0200909 struct sg_page_iter sg_iter;
Eric Anholt40123c12009-03-09 13:42:30 -0700910
Ville Syrjälä2bb46292013-02-22 16:12:51 +0200911 user_data = to_user_ptr(args->data_ptr);
Eric Anholt40123c12009-03-09 13:42:30 -0700912 remain = args->size;
913
Daniel Vetter8c599672011-12-14 13:57:31 +0100914 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
Eric Anholt40123c12009-03-09 13:42:30 -0700915
Daniel Vetter58642882012-03-25 19:47:37 +0200916 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
917 /* If we're not in the cpu write domain, set ourself into the gtt
918 * write domain and manually flush cachelines (if required). This
919 * optimizes for the case when the gpu will use the data
920 * right away and we therefore have to clflush anyway. */
Chris Wilson2c225692013-08-09 12:26:45 +0100921 needs_clflush_after = cpu_write_needs_clflush(obj);
Ben Widawsky23f54482013-09-11 14:57:48 -0700922 ret = i915_gem_object_wait_rendering(obj, false);
923 if (ret)
924 return ret;
Daniel Vetter58642882012-03-25 19:47:37 +0200925 }
Chris Wilsonc76ce032013-08-08 14:41:03 +0100926 /* Same trick applies to invalidate partially written cachelines read
927 * before writing. */
928 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
929 needs_clflush_before =
930 !cpu_cache_is_coherent(dev, obj->cache_level);
Daniel Vetter58642882012-03-25 19:47:37 +0200931
Chris Wilson755d2212012-09-04 21:02:55 +0100932 ret = i915_gem_object_get_pages(obj);
933 if (ret)
934 return ret;
935
Rodrigo Vivi77a0d1c2015-06-18 11:43:24 -0700936 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
Paulo Zanoni063e4e62015-02-13 17:23:45 -0200937
Chris Wilson755d2212012-09-04 21:02:55 +0100938 i915_gem_object_pin_pages(obj);
939
Eric Anholt40123c12009-03-09 13:42:30 -0700940 offset = args->offset;
Chris Wilson05394f32010-11-08 19:18:58 +0000941 obj->dirty = 1;
Eric Anholt40123c12009-03-09 13:42:30 -0700942
Imre Deak67d5a502013-02-18 19:28:02 +0200943 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
944 offset >> PAGE_SHIFT) {
Imre Deak2db76d72013-03-26 15:14:18 +0200945 struct page *page = sg_page_iter_page(&sg_iter);
Daniel Vetter58642882012-03-25 19:47:37 +0200946 int partial_cacheline_write;
Chris Wilsone5281cc2010-10-28 13:45:36 +0100947
Chris Wilson9da3da62012-06-01 15:20:22 +0100948 if (remain <= 0)
949 break;
950
Eric Anholt40123c12009-03-09 13:42:30 -0700951 /* Operation in this page
952 *
Eric Anholt40123c12009-03-09 13:42:30 -0700953 * shmem_page_offset = offset within page in shmem file
Eric Anholt40123c12009-03-09 13:42:30 -0700954 * page_length = bytes to copy for this page
955 */
Chris Wilsonc8cbbb82011-05-12 22:17:11 +0100956 shmem_page_offset = offset_in_page(offset);
Eric Anholt40123c12009-03-09 13:42:30 -0700957
958 page_length = remain;
959 if ((shmem_page_offset + page_length) > PAGE_SIZE)
960 page_length = PAGE_SIZE - shmem_page_offset;
Eric Anholt40123c12009-03-09 13:42:30 -0700961
Daniel Vetter58642882012-03-25 19:47:37 +0200962 /* If we don't overwrite a cacheline completely we need to be
963 * careful to have up-to-date data by first clflushing. Don't
964 * overcomplicate things and flush the entire patch. */
965 partial_cacheline_write = needs_clflush_before &&
966 ((shmem_page_offset | page_length)
967 & (boot_cpu_data.x86_clflush_size - 1));
968
Daniel Vetter8c599672011-12-14 13:57:31 +0100969 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
970 (page_to_phys(page) & (1 << 17)) != 0;
971
Daniel Vetterd174bd62012-03-25 19:47:40 +0200972 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
973 user_data, page_do_bit17_swizzling,
974 partial_cacheline_write,
975 needs_clflush_after);
976 if (ret == 0)
977 goto next_page;
Eric Anholt40123c12009-03-09 13:42:30 -0700978
Daniel Vettere244a442012-03-25 19:47:28 +0200979 hit_slowpath = 1;
Daniel Vettere244a442012-03-25 19:47:28 +0200980 mutex_unlock(&dev->struct_mutex);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200981 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
982 user_data, page_do_bit17_swizzling,
983 partial_cacheline_write,
984 needs_clflush_after);
Eric Anholt40123c12009-03-09 13:42:30 -0700985
Daniel Vettere244a442012-03-25 19:47:28 +0200986 mutex_lock(&dev->struct_mutex);
Chris Wilson755d2212012-09-04 21:02:55 +0100987
Chris Wilson755d2212012-09-04 21:02:55 +0100988 if (ret)
Daniel Vetter8c599672011-12-14 13:57:31 +0100989 goto out;
Daniel Vetter8c599672011-12-14 13:57:31 +0100990
Chris Wilson17793c92014-03-07 08:30:36 +0000991next_page:
Eric Anholt40123c12009-03-09 13:42:30 -0700992 remain -= page_length;
Daniel Vetter8c599672011-12-14 13:57:31 +0100993 user_data += page_length;
Eric Anholt40123c12009-03-09 13:42:30 -0700994 offset += page_length;
995 }
996
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100997out:
Chris Wilson755d2212012-09-04 21:02:55 +0100998 i915_gem_object_unpin_pages(obj);
999
Daniel Vettere244a442012-03-25 19:47:28 +02001000 if (hit_slowpath) {
Daniel Vetter8dcf0152012-11-15 16:53:58 +01001001 /*
1002 * Fixup: Flush cpu caches in case we didn't flush the dirty
1003 * cachelines in-line while writing and the object moved
1004 * out of the cpu write domain while we've dropped the lock.
1005 */
1006 if (!needs_clflush_after &&
1007 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
Chris Wilson000433b2013-08-08 14:41:09 +01001008 if (i915_gem_clflush_object(obj, obj->pin_display))
Ville Syrjäläed75a552015-08-11 19:47:10 +03001009 needs_clflush_after = true;
Daniel Vettere244a442012-03-25 19:47:28 +02001010 }
Daniel Vetter8c599672011-12-14 13:57:31 +01001011 }
Eric Anholt40123c12009-03-09 13:42:30 -07001012
Daniel Vetter58642882012-03-25 19:47:37 +02001013 if (needs_clflush_after)
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001014 i915_gem_chipset_flush(dev);
Ville Syrjäläed75a552015-08-11 19:47:10 +03001015 else
1016 obj->cache_dirty = true;
Daniel Vetter58642882012-03-25 19:47:37 +02001017
Rodrigo Vivide152b62015-07-07 16:28:51 -07001018 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
Eric Anholt40123c12009-03-09 13:42:30 -07001019 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07001020}
1021
1022/**
1023 * Writes data to the object referenced by handle.
1024 *
1025 * On error, the contents of the buffer that were to be modified are undefined.
1026 */
1027int
1028i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
Chris Wilsonfbd5a262010-10-14 15:03:58 +01001029 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001030{
Imre Deak5d77d9c2014-11-12 16:40:35 +02001031 struct drm_i915_private *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07001032 struct drm_i915_gem_pwrite *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00001033 struct drm_i915_gem_object *obj;
Chris Wilson51311d02010-11-17 09:10:42 +00001034 int ret;
1035
1036 if (args->size == 0)
1037 return 0;
1038
1039 if (!access_ok(VERIFY_READ,
Ville Syrjälä2bb46292013-02-22 16:12:51 +02001040 to_user_ptr(args->data_ptr),
Chris Wilson51311d02010-11-17 09:10:42 +00001041 args->size))
1042 return -EFAULT;
1043
Jani Nikulad330a952014-01-21 11:24:25 +02001044 if (likely(!i915.prefault_disable)) {
Xiong Zhang0b74b502013-07-19 13:51:24 +08001045 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
1046 args->size);
1047 if (ret)
1048 return -EFAULT;
1049 }
Eric Anholt673a3942008-07-30 12:06:12 -07001050
Imre Deak5d77d9c2014-11-12 16:40:35 +02001051 intel_runtime_pm_get(dev_priv);
1052
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001053 ret = i915_mutex_lock_interruptible(dev);
1054 if (ret)
Imre Deak5d77d9c2014-11-12 16:40:35 +02001055 goto put_rpm;
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001056
Chris Wilson05394f32010-11-08 19:18:58 +00001057 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00001058 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001059 ret = -ENOENT;
1060 goto unlock;
1061 }
Eric Anholt673a3942008-07-30 12:06:12 -07001062
Chris Wilson7dcd2492010-09-26 20:21:44 +01001063 /* Bounds check destination. */
Chris Wilson05394f32010-11-08 19:18:58 +00001064 if (args->offset > obj->base.size ||
1065 args->size > obj->base.size - args->offset) {
Chris Wilsonce9d4192010-09-26 20:50:05 +01001066 ret = -EINVAL;
Chris Wilson35b62a82010-09-26 20:23:38 +01001067 goto out;
Chris Wilsonce9d4192010-09-26 20:50:05 +01001068 }
1069
Daniel Vetter1286ff72012-05-10 15:25:09 +02001070 /* prime objects have no backing filp to GEM pread/pwrite
1071 * pages from.
1072 */
1073 if (!obj->base.filp) {
1074 ret = -EINVAL;
1075 goto out;
1076 }
1077
Chris Wilsondb53a302011-02-03 11:57:46 +00001078 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1079
Daniel Vetter935aaa62012-03-25 19:47:35 +02001080 ret = -EFAULT;
Eric Anholt673a3942008-07-30 12:06:12 -07001081 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1082 * it would end up going through the fenced access, and we'll get
1083 * different detiling behavior between reading and writing.
1084 * pread/pwrite currently are reading and writing from the CPU
1085 * perspective, requiring manual detiling by the client.
1086 */
Chris Wilson2c225692013-08-09 12:26:45 +01001087 if (obj->tiling_mode == I915_TILING_NONE &&
1088 obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
1089 cpu_write_needs_clflush(obj)) {
Chris Wilsonfbd5a262010-10-14 15:03:58 +01001090 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
Daniel Vetter935aaa62012-03-25 19:47:35 +02001091 /* Note that the gtt paths might fail with non-page-backed user
1092 * pointers (e.g. gtt mappings when moving data between
1093 * textures). Fallback to the shmem path in that case. */
Eric Anholt40123c12009-03-09 13:42:30 -07001094 }
Eric Anholt673a3942008-07-30 12:06:12 -07001095
Chris Wilson6a2c4232014-11-04 04:51:40 -08001096 if (ret == -EFAULT || ret == -ENOSPC) {
1097 if (obj->phys_handle)
1098 ret = i915_gem_phys_pwrite(obj, args, file);
1099 else
1100 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1101 }
Daniel Vetter5c0480f2011-12-14 13:57:30 +01001102
Chris Wilson35b62a82010-09-26 20:23:38 +01001103out:
Chris Wilson05394f32010-11-08 19:18:58 +00001104 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001105unlock:
Chris Wilsonfbd5a262010-10-14 15:03:58 +01001106 mutex_unlock(&dev->struct_mutex);
Imre Deak5d77d9c2014-11-12 16:40:35 +02001107put_rpm:
1108 intel_runtime_pm_put(dev_priv);
1109
Eric Anholt673a3942008-07-30 12:06:12 -07001110 return ret;
1111}
1112
Chris Wilsonb3612372012-08-24 09:35:08 +01001113int
Daniel Vetter33196de2012-11-14 17:14:05 +01001114i915_gem_check_wedge(struct i915_gpu_error *error,
Chris Wilsonb3612372012-08-24 09:35:08 +01001115 bool interruptible)
1116{
Daniel Vetter1f83fee2012-11-15 17:17:22 +01001117 if (i915_reset_in_progress(error)) {
Chris Wilsonb3612372012-08-24 09:35:08 +01001118 /* Non-interruptible callers can't handle -EAGAIN, hence return
1119 * -EIO unconditionally for these. */
1120 if (!interruptible)
1121 return -EIO;
1122
Daniel Vetter1f83fee2012-11-15 17:17:22 +01001123 /* Recovery complete, but the reset failed ... */
1124 if (i915_terminally_wedged(error))
Chris Wilsonb3612372012-08-24 09:35:08 +01001125 return -EIO;
1126
McAulay, Alistair6689c162014-08-15 18:51:35 +01001127 /*
1128 * Check if GPU Reset is in progress - we need intel_ring_begin
1129 * to work properly to reinit the hw state while the gpu is
1130 * still marked as reset-in-progress. Handle this with a flag.
1131 */
1132 if (!error->reload_in_reset)
1133 return -EAGAIN;
Chris Wilsonb3612372012-08-24 09:35:08 +01001134 }
1135
1136 return 0;
1137}
1138
Chris Wilson094f9a52013-09-25 17:34:55 +01001139static void fake_irq(unsigned long data)
1140{
1141 wake_up_process((struct task_struct *)data);
1142}
1143
1144static bool missed_irq(struct drm_i915_private *dev_priv,
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00001145 struct intel_engine_cs *engine)
Chris Wilson094f9a52013-09-25 17:34:55 +01001146{
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00001147 return test_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings);
Chris Wilson094f9a52013-09-25 17:34:55 +01001148}
1149
Chris Wilsonca5b7212015-12-11 11:32:58 +00001150static unsigned long local_clock_us(unsigned *cpu)
1151{
1152 unsigned long t;
1153
1154 /* Cheaply and approximately convert from nanoseconds to microseconds.
1155 * The result and subsequent calculations are also defined in the same
1156 * approximate microseconds units. The principal source of timing
1157 * error here is from the simple truncation.
1158 *
1159 * Note that local_clock() is only defined wrt to the current CPU;
1160 * the comparisons are no longer valid if we switch CPUs. Instead of
1161 * blocking preemption for the entire busywait, we can detect the CPU
1162 * switch and use that as indicator of system load and a reason to
1163 * stop busywaiting, see busywait_stop().
1164 */
1165 *cpu = get_cpu();
1166 t = local_clock() >> 10;
1167 put_cpu();
1168
1169 return t;
1170}
1171
1172static bool busywait_stop(unsigned long timeout, unsigned cpu)
1173{
1174 unsigned this_cpu;
1175
1176 if (time_after(local_clock_us(&this_cpu), timeout))
1177 return true;
1178
1179 return this_cpu != cpu;
1180}
1181
Chris Wilson91b0c352015-12-11 11:32:57 +00001182static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
Chris Wilsonb29c19b2013-09-25 17:34:56 +01001183{
Chris Wilson2def4ad92015-04-07 16:20:41 +01001184 unsigned long timeout;
Chris Wilsonca5b7212015-12-11 11:32:58 +00001185 unsigned cpu;
Chris Wilsonb29c19b2013-09-25 17:34:56 +01001186
Chris Wilsonca5b7212015-12-11 11:32:58 +00001187 /* When waiting for high frequency requests, e.g. during synchronous
1188 * rendering split between the CPU and GPU, the finite amount of time
1189 * required to set up the irq and wait upon it limits the response
1190 * rate. By busywaiting on the request completion for a short while we
1191 * can service the high frequency waits as quick as possible. However,
1192 * if it is a slow request, we want to sleep as quickly as possible.
1193 * The tradeoff between waiting and sleeping is roughly the time it
1194 * takes to sleep on a request, on the order of a microsecond.
1195 */
Chris Wilsonb29c19b2013-09-25 17:34:56 +01001196
Tvrtko Ursulin4a570db2016-03-16 11:00:38 +00001197 if (req->engine->irq_refcount)
Chris Wilson2def4ad92015-04-07 16:20:41 +01001198 return -EBUSY;
1199
Chris Wilson821485d2015-12-11 11:32:59 +00001200 /* Only spin if we know the GPU is processing this request */
1201 if (!i915_gem_request_started(req, true))
1202 return -EAGAIN;
1203
Chris Wilsonca5b7212015-12-11 11:32:58 +00001204 timeout = local_clock_us(&cpu) + 5;
Chris Wilson2def4ad92015-04-07 16:20:41 +01001205 while (!need_resched()) {
Daniel Vettereed29a52015-05-21 14:21:25 +02001206 if (i915_gem_request_completed(req, true))
Chris Wilson2def4ad92015-04-07 16:20:41 +01001207 return 0;
1208
Chris Wilson91b0c352015-12-11 11:32:57 +00001209 if (signal_pending_state(state, current))
1210 break;
1211
Chris Wilsonca5b7212015-12-11 11:32:58 +00001212 if (busywait_stop(timeout, cpu))
Chris Wilson2def4ad92015-04-07 16:20:41 +01001213 break;
1214
1215 cpu_relax_lowlatency();
1216 }
Chris Wilson821485d2015-12-11 11:32:59 +00001217
Daniel Vettereed29a52015-05-21 14:21:25 +02001218 if (i915_gem_request_completed(req, false))
Chris Wilson2def4ad92015-04-07 16:20:41 +01001219 return 0;
1220
1221 return -EAGAIN;
Chris Wilsonb29c19b2013-09-25 17:34:56 +01001222}
1223
Chris Wilsonb3612372012-08-24 09:35:08 +01001224/**
John Harrison9c654812014-11-24 18:49:35 +00001225 * __i915_wait_request - wait until execution of request has finished
1226 * @req: duh!
1227 * @reset_counter: reset sequence associated with the given request
Chris Wilsonb3612372012-08-24 09:35:08 +01001228 * @interruptible: do an interruptible wait (normally yes)
1229 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1230 *
Daniel Vetterf69061b2012-12-06 09:01:42 +01001231 * Note: It is of utmost importance that the passed in seqno and reset_counter
1232 * values have been read by the caller in an smp safe manner. Where read-side
1233 * locks are involved, it is sufficient to read the reset_counter before
1234 * unlocking the lock that protects the seqno. For lockless tricks, the
1235 * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1236 * inserted.
1237 *
John Harrison9c654812014-11-24 18:49:35 +00001238 * Returns 0 if the request was found within the alloted time. Else returns the
Chris Wilsonb3612372012-08-24 09:35:08 +01001239 * errno with remaining time filled in timeout argument.
1240 */
John Harrison9c654812014-11-24 18:49:35 +00001241int __i915_wait_request(struct drm_i915_gem_request *req,
Daniel Vetterf69061b2012-12-06 09:01:42 +01001242 unsigned reset_counter,
Chris Wilsonb29c19b2013-09-25 17:34:56 +01001243 bool interruptible,
Thomas Gleixner5ed0bdf2014-07-16 21:05:06 +00001244 s64 *timeout,
Chris Wilson2e1b8732015-04-27 13:41:22 +01001245 struct intel_rps_client *rps)
Chris Wilsonb3612372012-08-24 09:35:08 +01001246{
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00001247 struct intel_engine_cs *engine = i915_gem_request_get_engine(req);
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00001248 struct drm_device *dev = engine->dev;
Jani Nikula3e31c6c2014-03-31 14:27:16 +03001249 struct drm_i915_private *dev_priv = dev->dev_private;
Mika Kuoppala168c3f22013-12-12 17:54:42 +02001250 const bool irq_test_in_progress =
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00001251 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_engine_flag(engine);
Chris Wilson91b0c352015-12-11 11:32:57 +00001252 int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
Chris Wilson094f9a52013-09-25 17:34:55 +01001253 DEFINE_WAIT(wait);
Mika Kuoppala47e9766d2013-12-10 17:02:43 +02001254 unsigned long timeout_expire;
Tvrtko Ursuline0313db2016-01-15 15:11:12 +00001255 s64 before = 0; /* Only to silence a compiler warning. */
Chris Wilsonb3612372012-08-24 09:35:08 +01001256 int ret;
1257
Jesse Barnes9df7575f2014-06-20 09:29:20 -07001258 WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
Paulo Zanonic67a4702013-08-19 13:18:09 -03001259
Chris Wilsonb4716182015-04-27 13:41:17 +01001260 if (list_empty(&req->list))
1261 return 0;
1262
John Harrison1b5a4332014-11-24 18:49:42 +00001263 if (i915_gem_request_completed(req, true))
Chris Wilsonb3612372012-08-24 09:35:08 +01001264 return 0;
1265
Chris Wilsonbb6d1982015-11-26 13:31:42 +00001266 timeout_expire = 0;
1267 if (timeout) {
1268 if (WARN_ON(*timeout < 0))
1269 return -EINVAL;
1270
1271 if (*timeout == 0)
1272 return -ETIME;
1273
1274 timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout);
Tvrtko Ursuline0313db2016-01-15 15:11:12 +00001275
1276 /*
1277 * Record current time in case interrupted by signal, or wedged.
1278 */
1279 before = ktime_get_raw_ns();
Chris Wilsonbb6d1982015-11-26 13:31:42 +00001280 }
Chris Wilsonb3612372012-08-24 09:35:08 +01001281
Chris Wilson2e1b8732015-04-27 13:41:22 +01001282 if (INTEL_INFO(dev_priv)->gen >= 6)
Chris Wilsone61b9952015-04-27 13:41:24 +01001283 gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
Chris Wilsonb3612372012-08-24 09:35:08 +01001284
John Harrison74328ee2014-11-24 18:49:38 +00001285 trace_i915_gem_request_wait_begin(req);
Chris Wilson2def4ad92015-04-07 16:20:41 +01001286
1287 /* Optimistic spin for the next jiffie before touching IRQs */
Chris Wilson91b0c352015-12-11 11:32:57 +00001288 ret = __i915_spin_request(req, state);
Chris Wilson2def4ad92015-04-07 16:20:41 +01001289 if (ret == 0)
1290 goto out;
1291
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00001292 if (!irq_test_in_progress && WARN_ON(!engine->irq_get(engine))) {
Chris Wilson2def4ad92015-04-07 16:20:41 +01001293 ret = -ENODEV;
1294 goto out;
1295 }
1296
Chris Wilson094f9a52013-09-25 17:34:55 +01001297 for (;;) {
1298 struct timer_list timer;
Chris Wilsonb3612372012-08-24 09:35:08 +01001299
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00001300 prepare_to_wait(&engine->irq_queue, &wait, state);
Chris Wilsonb3612372012-08-24 09:35:08 +01001301
Daniel Vetterf69061b2012-12-06 09:01:42 +01001302 /* We need to check whether any gpu reset happened in between
1303 * the caller grabbing the seqno and now ... */
Chris Wilson094f9a52013-09-25 17:34:55 +01001304 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
1305 /* ... but upgrade the -EAGAIN to an -EIO if the gpu
1306 * is truely gone. */
1307 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1308 if (ret == 0)
1309 ret = -EAGAIN;
1310 break;
1311 }
Daniel Vetterf69061b2012-12-06 09:01:42 +01001312
John Harrison1b5a4332014-11-24 18:49:42 +00001313 if (i915_gem_request_completed(req, false)) {
Chris Wilson094f9a52013-09-25 17:34:55 +01001314 ret = 0;
1315 break;
1316 }
Chris Wilsonb3612372012-08-24 09:35:08 +01001317
Chris Wilson91b0c352015-12-11 11:32:57 +00001318 if (signal_pending_state(state, current)) {
Chris Wilson094f9a52013-09-25 17:34:55 +01001319 ret = -ERESTARTSYS;
1320 break;
1321 }
1322
Mika Kuoppala47e9766d2013-12-10 17:02:43 +02001323 if (timeout && time_after_eq(jiffies, timeout_expire)) {
Chris Wilson094f9a52013-09-25 17:34:55 +01001324 ret = -ETIME;
1325 break;
1326 }
1327
1328 timer.function = NULL;
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00001329 if (timeout || missed_irq(dev_priv, engine)) {
Mika Kuoppala47e9766d2013-12-10 17:02:43 +02001330 unsigned long expire;
1331
Chris Wilson094f9a52013-09-25 17:34:55 +01001332 setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00001333 expire = missed_irq(dev_priv, engine) ? jiffies + 1 : timeout_expire;
Chris Wilson094f9a52013-09-25 17:34:55 +01001334 mod_timer(&timer, expire);
1335 }
1336
Chris Wilson5035c272013-10-04 09:58:46 +01001337 io_schedule();
Chris Wilson094f9a52013-09-25 17:34:55 +01001338
Chris Wilson094f9a52013-09-25 17:34:55 +01001339 if (timer.function) {
1340 del_singleshot_timer_sync(&timer);
1341 destroy_timer_on_stack(&timer);
1342 }
1343 }
Mika Kuoppala168c3f22013-12-12 17:54:42 +02001344 if (!irq_test_in_progress)
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00001345 engine->irq_put(engine);
Chris Wilson094f9a52013-09-25 17:34:55 +01001346
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00001347 finish_wait(&engine->irq_queue, &wait);
Chris Wilsonb3612372012-08-24 09:35:08 +01001348
Chris Wilson2def4ad92015-04-07 16:20:41 +01001349out:
Chris Wilson2def4ad92015-04-07 16:20:41 +01001350 trace_i915_gem_request_wait_end(req);
1351
Chris Wilsonb3612372012-08-24 09:35:08 +01001352 if (timeout) {
Tvrtko Ursuline0313db2016-01-15 15:11:12 +00001353 s64 tres = *timeout - (ktime_get_raw_ns() - before);
Thomas Gleixner5ed0bdf2014-07-16 21:05:06 +00001354
1355 *timeout = tres < 0 ? 0 : tres;
Daniel Vetter9cca3062014-11-28 10:29:55 +01001356
1357 /*
1358 * Apparently ktime isn't accurate enough and occasionally has a
1359 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
1360 * things up to make the test happy. We allow up to 1 jiffy.
1361 *
1362 * This is a regrssion from the timespec->ktime conversion.
1363 */
1364 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
1365 *timeout = 0;
Chris Wilsonb3612372012-08-24 09:35:08 +01001366 }
1367
Chris Wilson094f9a52013-09-25 17:34:55 +01001368 return ret;
Chris Wilsonb3612372012-08-24 09:35:08 +01001369}
1370
John Harrisonfcfa423c2015-05-29 17:44:12 +01001371int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
1372 struct drm_file *file)
1373{
John Harrisonfcfa423c2015-05-29 17:44:12 +01001374 struct drm_i915_file_private *file_priv;
1375
1376 WARN_ON(!req || !file || req->file_priv);
1377
1378 if (!req || !file)
1379 return -EINVAL;
1380
1381 if (req->file_priv)
1382 return -EINVAL;
1383
John Harrisonfcfa423c2015-05-29 17:44:12 +01001384 file_priv = file->driver_priv;
1385
1386 spin_lock(&file_priv->mm.lock);
1387 req->file_priv = file_priv;
1388 list_add_tail(&req->client_list, &file_priv->mm.request_list);
1389 spin_unlock(&file_priv->mm.lock);
1390
1391 req->pid = get_pid(task_pid(current));
1392
1393 return 0;
1394}
1395
Chris Wilsonb4716182015-04-27 13:41:17 +01001396static inline void
1397i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1398{
1399 struct drm_i915_file_private *file_priv = request->file_priv;
1400
1401 if (!file_priv)
1402 return;
1403
1404 spin_lock(&file_priv->mm.lock);
1405 list_del(&request->client_list);
1406 request->file_priv = NULL;
1407 spin_unlock(&file_priv->mm.lock);
John Harrisonfcfa423c2015-05-29 17:44:12 +01001408
1409 put_pid(request->pid);
1410 request->pid = NULL;
Chris Wilsonb4716182015-04-27 13:41:17 +01001411}
1412
1413static void i915_gem_request_retire(struct drm_i915_gem_request *request)
1414{
1415 trace_i915_gem_request_retire(request);
1416
1417 /* We know the GPU must have read the request to have
1418 * sent us the seqno + interrupt, so use the position
1419 * of tail of the request to update the last known position
1420 * of the GPU head.
1421 *
1422 * Note this requires that we are always called in request
1423 * completion order.
1424 */
1425 request->ringbuf->last_retired_head = request->postfix;
1426
1427 list_del_init(&request->list);
1428 i915_gem_request_remove_from_client(request);
1429
Chris Wilsonb4716182015-04-27 13:41:17 +01001430 i915_gem_request_unreference(request);
1431}
1432
1433static void
1434__i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
1435{
Tvrtko Ursulin4a570db2016-03-16 11:00:38 +00001436 struct intel_engine_cs *engine = req->engine;
Chris Wilsonb4716182015-04-27 13:41:17 +01001437 struct drm_i915_gem_request *tmp;
1438
1439 lockdep_assert_held(&engine->dev->struct_mutex);
1440
1441 if (list_empty(&req->list))
1442 return;
1443
1444 do {
1445 tmp = list_first_entry(&engine->request_list,
1446 typeof(*tmp), list);
1447
1448 i915_gem_request_retire(tmp);
1449 } while (tmp != req);
1450
1451 WARN_ON(i915_verify_lists(engine->dev));
1452}
1453
Chris Wilsonb3612372012-08-24 09:35:08 +01001454/**
Daniel Vettera4b3a572014-11-26 14:17:05 +01001455 * Waits for a request to be signaled, and cleans up the
Chris Wilsonb3612372012-08-24 09:35:08 +01001456 * request and object lists appropriately for that event.
1457 */
1458int
Daniel Vettera4b3a572014-11-26 14:17:05 +01001459i915_wait_request(struct drm_i915_gem_request *req)
Chris Wilsonb3612372012-08-24 09:35:08 +01001460{
Daniel Vettera4b3a572014-11-26 14:17:05 +01001461 struct drm_device *dev;
1462 struct drm_i915_private *dev_priv;
1463 bool interruptible;
Chris Wilsonb3612372012-08-24 09:35:08 +01001464 int ret;
1465
Daniel Vettera4b3a572014-11-26 14:17:05 +01001466 BUG_ON(req == NULL);
1467
Tvrtko Ursulin4a570db2016-03-16 11:00:38 +00001468 dev = req->engine->dev;
Daniel Vettera4b3a572014-11-26 14:17:05 +01001469 dev_priv = dev->dev_private;
1470 interruptible = dev_priv->mm.interruptible;
1471
Chris Wilsonb3612372012-08-24 09:35:08 +01001472 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
Chris Wilsonb3612372012-08-24 09:35:08 +01001473
Daniel Vetter33196de2012-11-14 17:14:05 +01001474 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
Chris Wilsonb3612372012-08-24 09:35:08 +01001475 if (ret)
1476 return ret;
1477
Chris Wilsonb4716182015-04-27 13:41:17 +01001478 ret = __i915_wait_request(req,
1479 atomic_read(&dev_priv->gpu_error.reset_counter),
John Harrison9c654812014-11-24 18:49:35 +00001480 interruptible, NULL, NULL);
Chris Wilsonb4716182015-04-27 13:41:17 +01001481 if (ret)
1482 return ret;
Chris Wilsonb3612372012-08-24 09:35:08 +01001483
Chris Wilsonb4716182015-04-27 13:41:17 +01001484 __i915_gem_request_retire__upto(req);
Chris Wilsond26e3af2013-06-29 22:05:26 +01001485 return 0;
1486}
1487
Chris Wilsonb3612372012-08-24 09:35:08 +01001488/**
1489 * Ensures that all rendering to the object has completed and the object is
1490 * safe to unbind from the GTT or access from the CPU.
1491 */
Chris Wilson2e2f3512015-04-27 13:41:14 +01001492int
Chris Wilsonb3612372012-08-24 09:35:08 +01001493i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1494 bool readonly)
1495{
Chris Wilsonb4716182015-04-27 13:41:17 +01001496 int ret, i;
Chris Wilsonb3612372012-08-24 09:35:08 +01001497
Chris Wilsonb4716182015-04-27 13:41:17 +01001498 if (!obj->active)
Chris Wilsonb3612372012-08-24 09:35:08 +01001499 return 0;
1500
Chris Wilsonb4716182015-04-27 13:41:17 +01001501 if (readonly) {
1502 if (obj->last_write_req != NULL) {
1503 ret = i915_wait_request(obj->last_write_req);
1504 if (ret)
1505 return ret;
Chris Wilsonb3612372012-08-24 09:35:08 +01001506
Tvrtko Ursulin4a570db2016-03-16 11:00:38 +00001507 i = obj->last_write_req->engine->id;
Chris Wilsonb4716182015-04-27 13:41:17 +01001508 if (obj->last_read_req[i] == obj->last_write_req)
1509 i915_gem_object_retire__read(obj, i);
1510 else
1511 i915_gem_object_retire__write(obj);
1512 }
1513 } else {
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00001514 for (i = 0; i < I915_NUM_ENGINES; i++) {
Chris Wilsonb4716182015-04-27 13:41:17 +01001515 if (obj->last_read_req[i] == NULL)
1516 continue;
1517
1518 ret = i915_wait_request(obj->last_read_req[i]);
1519 if (ret)
1520 return ret;
1521
1522 i915_gem_object_retire__read(obj, i);
1523 }
1524 RQ_BUG_ON(obj->active);
1525 }
1526
1527 return 0;
1528}
1529
1530static void
1531i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
1532 struct drm_i915_gem_request *req)
1533{
Tvrtko Ursulin4a570db2016-03-16 11:00:38 +00001534 int ring = req->engine->id;
Chris Wilsonb4716182015-04-27 13:41:17 +01001535
1536 if (obj->last_read_req[ring] == req)
1537 i915_gem_object_retire__read(obj, ring);
1538 else if (obj->last_write_req == req)
1539 i915_gem_object_retire__write(obj);
1540
1541 __i915_gem_request_retire__upto(req);
Chris Wilsonb3612372012-08-24 09:35:08 +01001542}
1543
Chris Wilson3236f572012-08-24 09:35:09 +01001544/* A nonblocking variant of the above wait. This is a highly dangerous routine
1545 * as the object state may change during this call.
1546 */
1547static __must_check int
1548i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
Chris Wilson2e1b8732015-04-27 13:41:22 +01001549 struct intel_rps_client *rps,
Chris Wilson3236f572012-08-24 09:35:09 +01001550 bool readonly)
1551{
1552 struct drm_device *dev = obj->base.dev;
1553 struct drm_i915_private *dev_priv = dev->dev_private;
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00001554 struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
Daniel Vetterf69061b2012-12-06 09:01:42 +01001555 unsigned reset_counter;
Chris Wilsonb4716182015-04-27 13:41:17 +01001556 int ret, i, n = 0;
Chris Wilson3236f572012-08-24 09:35:09 +01001557
1558 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1559 BUG_ON(!dev_priv->mm.interruptible);
1560
Chris Wilsonb4716182015-04-27 13:41:17 +01001561 if (!obj->active)
Chris Wilson3236f572012-08-24 09:35:09 +01001562 return 0;
1563
Daniel Vetter33196de2012-11-14 17:14:05 +01001564 ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
Chris Wilson3236f572012-08-24 09:35:09 +01001565 if (ret)
1566 return ret;
1567
Daniel Vetterf69061b2012-12-06 09:01:42 +01001568 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
Chris Wilson3236f572012-08-24 09:35:09 +01001569
Chris Wilsonb4716182015-04-27 13:41:17 +01001570 if (readonly) {
1571 struct drm_i915_gem_request *req;
1572
1573 req = obj->last_write_req;
1574 if (req == NULL)
1575 return 0;
1576
Chris Wilsonb4716182015-04-27 13:41:17 +01001577 requests[n++] = i915_gem_request_reference(req);
1578 } else {
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00001579 for (i = 0; i < I915_NUM_ENGINES; i++) {
Chris Wilsonb4716182015-04-27 13:41:17 +01001580 struct drm_i915_gem_request *req;
1581
1582 req = obj->last_read_req[i];
1583 if (req == NULL)
1584 continue;
1585
Chris Wilsonb4716182015-04-27 13:41:17 +01001586 requests[n++] = i915_gem_request_reference(req);
1587 }
1588 }
1589
1590 mutex_unlock(&dev->struct_mutex);
1591 for (i = 0; ret == 0 && i < n; i++)
1592 ret = __i915_wait_request(requests[i], reset_counter, true,
Chris Wilson2e1b8732015-04-27 13:41:22 +01001593 NULL, rps);
Chris Wilsonb4716182015-04-27 13:41:17 +01001594 mutex_lock(&dev->struct_mutex);
1595
Chris Wilsonb4716182015-04-27 13:41:17 +01001596 for (i = 0; i < n; i++) {
1597 if (ret == 0)
1598 i915_gem_object_retire_request(obj, requests[i]);
1599 i915_gem_request_unreference(requests[i]);
1600 }
1601
1602 return ret;
Chris Wilson3236f572012-08-24 09:35:09 +01001603}
1604
Chris Wilson2e1b8732015-04-27 13:41:22 +01001605static struct intel_rps_client *to_rps_client(struct drm_file *file)
1606{
1607 struct drm_i915_file_private *fpriv = file->driver_priv;
1608 return &fpriv->rps;
1609}
1610
Eric Anholt673a3942008-07-30 12:06:12 -07001611/**
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001612 * Called when user space prepares to use an object with the CPU, either
1613 * through the mmap ioctl's mapping or a GTT mapping.
Eric Anholt673a3942008-07-30 12:06:12 -07001614 */
1615int
1616i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001617 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001618{
1619 struct drm_i915_gem_set_domain *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00001620 struct drm_i915_gem_object *obj;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001621 uint32_t read_domains = args->read_domains;
1622 uint32_t write_domain = args->write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -07001623 int ret;
1624
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001625 /* Only handle setting domains to types used by the CPU. */
Chris Wilson21d509e2009-06-06 09:46:02 +01001626 if (write_domain & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001627 return -EINVAL;
1628
Chris Wilson21d509e2009-06-06 09:46:02 +01001629 if (read_domains & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001630 return -EINVAL;
1631
1632 /* Having something in the write domain implies it's in the read
1633 * domain, and only that read domain. Enforce that in the request.
1634 */
1635 if (write_domain != 0 && read_domains != write_domain)
1636 return -EINVAL;
1637
Chris Wilson76c1dec2010-09-25 11:22:51 +01001638 ret = i915_mutex_lock_interruptible(dev);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001639 if (ret)
Chris Wilson76c1dec2010-09-25 11:22:51 +01001640 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07001641
Chris Wilson05394f32010-11-08 19:18:58 +00001642 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00001643 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001644 ret = -ENOENT;
1645 goto unlock;
Chris Wilson76c1dec2010-09-25 11:22:51 +01001646 }
Jesse Barnes652c3932009-08-17 13:31:43 -07001647
Chris Wilson3236f572012-08-24 09:35:09 +01001648 /* Try to flush the object off the GPU without holding the lock.
1649 * We will repeat the flush holding the lock in the normal manner
1650 * to catch cases where we are gazumped.
1651 */
Chris Wilson6e4930f2014-02-07 18:37:06 -02001652 ret = i915_gem_object_wait_rendering__nonblocking(obj,
Chris Wilson2e1b8732015-04-27 13:41:22 +01001653 to_rps_client(file),
Chris Wilson6e4930f2014-02-07 18:37:06 -02001654 !write_domain);
Chris Wilson3236f572012-08-24 09:35:09 +01001655 if (ret)
1656 goto unref;
1657
Chris Wilson43566de2015-01-02 16:29:29 +05301658 if (read_domains & I915_GEM_DOMAIN_GTT)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001659 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
Chris Wilson43566de2015-01-02 16:29:29 +05301660 else
Eric Anholte47c68e2008-11-14 13:35:19 -08001661 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001662
Daniel Vetter031b6982015-06-26 19:35:16 +02001663 if (write_domain != 0)
1664 intel_fb_obj_invalidate(obj,
1665 write_domain == I915_GEM_DOMAIN_GTT ?
1666 ORIGIN_GTT : ORIGIN_CPU);
1667
Chris Wilson3236f572012-08-24 09:35:09 +01001668unref:
Chris Wilson05394f32010-11-08 19:18:58 +00001669 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001670unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07001671 mutex_unlock(&dev->struct_mutex);
1672 return ret;
1673}
1674
1675/**
1676 * Called when user space has done writes to this buffer
1677 */
1678int
1679i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001680 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001681{
1682 struct drm_i915_gem_sw_finish *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00001683 struct drm_i915_gem_object *obj;
Eric Anholt673a3942008-07-30 12:06:12 -07001684 int ret = 0;
1685
Chris Wilson76c1dec2010-09-25 11:22:51 +01001686 ret = i915_mutex_lock_interruptible(dev);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001687 if (ret)
Chris Wilson76c1dec2010-09-25 11:22:51 +01001688 return ret;
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001689
Chris Wilson05394f32010-11-08 19:18:58 +00001690 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00001691 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001692 ret = -ENOENT;
1693 goto unlock;
Eric Anholt673a3942008-07-30 12:06:12 -07001694 }
1695
Eric Anholt673a3942008-07-30 12:06:12 -07001696 /* Pinned buffers may be scanout, so flush the cache */
Chris Wilson2c225692013-08-09 12:26:45 +01001697 if (obj->pin_display)
Daniel Vettere62b59e2015-01-21 14:53:48 +01001698 i915_gem_object_flush_cpu_write_domain(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08001699
Chris Wilson05394f32010-11-08 19:18:58 +00001700 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001701unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07001702 mutex_unlock(&dev->struct_mutex);
1703 return ret;
1704}
1705
1706/**
1707 * Maps the contents of an object, returning the address it is mapped
1708 * into.
1709 *
1710 * While the mapping holds a reference on the contents of the object, it doesn't
1711 * imply a ref on the object itself.
Daniel Vetter34367382014-10-16 12:28:18 +02001712 *
1713 * IMPORTANT:
1714 *
1715 * DRM driver writers who look a this function as an example for how to do GEM
1716 * mmap support, please don't implement mmap support like here. The modern way
1717 * to implement DRM mmap support is with an mmap offset ioctl (like
1718 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1719 * That way debug tooling like valgrind will understand what's going on, hiding
1720 * the mmap call in a driver private ioctl will break that. The i915 driver only
1721 * does cpu mmaps this way because we didn't know better.
Eric Anholt673a3942008-07-30 12:06:12 -07001722 */
1723int
1724i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001725 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001726{
1727 struct drm_i915_gem_mmap *args = data;
1728 struct drm_gem_object *obj;
Eric Anholt673a3942008-07-30 12:06:12 -07001729 unsigned long addr;
1730
Akash Goel1816f922015-01-02 16:29:30 +05301731 if (args->flags & ~(I915_MMAP_WC))
1732 return -EINVAL;
1733
1734 if (args->flags & I915_MMAP_WC && !cpu_has_pat)
1735 return -ENODEV;
1736
Chris Wilson05394f32010-11-08 19:18:58 +00001737 obj = drm_gem_object_lookup(dev, file, args->handle);
Eric Anholt673a3942008-07-30 12:06:12 -07001738 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +01001739 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07001740
Daniel Vetter1286ff72012-05-10 15:25:09 +02001741 /* prime objects have no backing filp to GEM mmap
1742 * pages from.
1743 */
1744 if (!obj->filp) {
1745 drm_gem_object_unreference_unlocked(obj);
1746 return -EINVAL;
1747 }
1748
Linus Torvalds6be5ceb2012-04-20 17:13:58 -07001749 addr = vm_mmap(obj->filp, 0, args->size,
Eric Anholt673a3942008-07-30 12:06:12 -07001750 PROT_READ | PROT_WRITE, MAP_SHARED,
1751 args->offset);
Akash Goel1816f922015-01-02 16:29:30 +05301752 if (args->flags & I915_MMAP_WC) {
1753 struct mm_struct *mm = current->mm;
1754 struct vm_area_struct *vma;
1755
1756 down_write(&mm->mmap_sem);
1757 vma = find_vma(mm, addr);
1758 if (vma)
1759 vma->vm_page_prot =
1760 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1761 else
1762 addr = -ENOMEM;
1763 up_write(&mm->mmap_sem);
1764 }
Luca Barbieribc9025b2010-02-09 05:49:12 +00001765 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001766 if (IS_ERR((void *)addr))
1767 return addr;
1768
1769 args->addr_ptr = (uint64_t) addr;
1770
1771 return 0;
1772}
1773
Jesse Barnesde151cf2008-11-12 10:03:55 -08001774/**
1775 * i915_gem_fault - fault a page into the GTT
Geliang Tangd9072a32015-09-15 05:58:44 -07001776 * @vma: VMA in question
1777 * @vmf: fault info
Jesse Barnesde151cf2008-11-12 10:03:55 -08001778 *
1779 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1780 * from userspace. The fault handler takes care of binding the object to
1781 * the GTT (if needed), allocating and programming a fence register (again,
1782 * only if needed based on whether the old reg is still valid or the object
1783 * is tiled) and inserting a new PTE into the faulting process.
1784 *
1785 * Note that the faulting process may involve evicting existing objects
1786 * from the GTT and/or fence registers to make room. So performance may
1787 * suffer if the GTT working set is large or there are few fence registers
1788 * left.
1789 */
1790int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1791{
Chris Wilson05394f32010-11-08 19:18:58 +00001792 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1793 struct drm_device *dev = obj->base.dev;
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03001794 struct drm_i915_private *dev_priv = to_i915(dev);
1795 struct i915_ggtt *ggtt = &dev_priv->ggtt;
Joonas Lahtinenc5ad54c2015-05-06 14:36:09 +03001796 struct i915_ggtt_view view = i915_ggtt_view_normal;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001797 pgoff_t page_offset;
1798 unsigned long pfn;
1799 int ret = 0;
Jesse Barnes0f973f22009-01-26 17:10:45 -08001800 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001801
Paulo Zanonif65c9162013-11-27 18:20:34 -02001802 intel_runtime_pm_get(dev_priv);
1803
Jesse Barnesde151cf2008-11-12 10:03:55 -08001804 /* We don't use vmf->pgoff since that has the fake offset */
1805 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1806 PAGE_SHIFT;
1807
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001808 ret = i915_mutex_lock_interruptible(dev);
1809 if (ret)
1810 goto out;
Chris Wilsona00b10c2010-09-24 21:15:47 +01001811
Chris Wilsondb53a302011-02-03 11:57:46 +00001812 trace_i915_gem_object_fault(obj, page_offset, true, write);
1813
Chris Wilson6e4930f2014-02-07 18:37:06 -02001814 /* Try to flush the object off the GPU first without holding the lock.
1815 * Upon reacquiring the lock, we will perform our sanity checks and then
1816 * repeat the flush holding the lock in the normal manner to catch cases
1817 * where we are gazumped.
1818 */
1819 ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1820 if (ret)
1821 goto unlock;
1822
Chris Wilsoneb119bd2012-12-16 12:43:36 +00001823 /* Access to snoopable pages through the GTT is incoherent. */
1824 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
Chris Wilsonddeff6e2014-05-28 16:16:41 +01001825 ret = -EFAULT;
Chris Wilsoneb119bd2012-12-16 12:43:36 +00001826 goto unlock;
1827 }
1828
Joonas Lahtinenc5ad54c2015-05-06 14:36:09 +03001829 /* Use a partial view if the object is bigger than the aperture. */
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03001830 if (obj->base.size >= ggtt->mappable_end &&
Joonas Lahtinene7ded2d2015-05-08 14:37:39 +03001831 obj->tiling_mode == I915_TILING_NONE) {
Joonas Lahtinenc5ad54c2015-05-06 14:36:09 +03001832 static const unsigned int chunk_size = 256; // 1 MiB
Joonas Lahtinene7ded2d2015-05-08 14:37:39 +03001833
Joonas Lahtinenc5ad54c2015-05-06 14:36:09 +03001834 memset(&view, 0, sizeof(view));
1835 view.type = I915_GGTT_VIEW_PARTIAL;
1836 view.params.partial.offset = rounddown(page_offset, chunk_size);
1837 view.params.partial.size =
1838 min_t(unsigned int,
1839 chunk_size,
1840 (vma->vm_end - vma->vm_start)/PAGE_SIZE -
1841 view.params.partial.offset);
1842 }
1843
1844 /* Now pin it into the GTT if needed */
1845 ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
Chris Wilsond9e86c02010-11-10 16:40:20 +00001846 if (ret)
1847 goto unlock;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001848
Chris Wilsonc9839302012-11-20 10:45:17 +00001849 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1850 if (ret)
1851 goto unpin;
1852
1853 ret = i915_gem_object_get_fence(obj);
1854 if (ret)
1855 goto unpin;
Chris Wilson7d1c4802010-08-07 21:45:03 +01001856
Chris Wilsonb90b91d2014-06-10 12:14:40 +01001857 /* Finally, remap it using the new GTT offset */
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03001858 pfn = ggtt->mappable_base +
Joonas Lahtinenc5ad54c2015-05-06 14:36:09 +03001859 i915_gem_obj_ggtt_offset_view(obj, &view);
Ben Widawskyf343c5f2013-07-05 14:41:04 -07001860 pfn >>= PAGE_SHIFT;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001861
Joonas Lahtinenc5ad54c2015-05-06 14:36:09 +03001862 if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
1863 /* Overriding existing pages in partial view does not cause
1864 * us any trouble as TLBs are still valid because the fault
1865 * is due to userspace losing part of the mapping or never
1866 * having accessed it before (at this partials' range).
1867 */
1868 unsigned long base = vma->vm_start +
1869 (view.params.partial.offset << PAGE_SHIFT);
1870 unsigned int i;
Chris Wilsonb90b91d2014-06-10 12:14:40 +01001871
Joonas Lahtinenc5ad54c2015-05-06 14:36:09 +03001872 for (i = 0; i < view.params.partial.size; i++) {
1873 ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
Chris Wilsonb90b91d2014-06-10 12:14:40 +01001874 if (ret)
1875 break;
1876 }
1877
1878 obj->fault_mappable = true;
Joonas Lahtinenc5ad54c2015-05-06 14:36:09 +03001879 } else {
1880 if (!obj->fault_mappable) {
1881 unsigned long size = min_t(unsigned long,
1882 vma->vm_end - vma->vm_start,
1883 obj->base.size);
1884 int i;
1885
1886 for (i = 0; i < size >> PAGE_SHIFT; i++) {
1887 ret = vm_insert_pfn(vma,
1888 (unsigned long)vma->vm_start + i * PAGE_SIZE,
1889 pfn + i);
1890 if (ret)
1891 break;
1892 }
1893
1894 obj->fault_mappable = true;
1895 } else
1896 ret = vm_insert_pfn(vma,
1897 (unsigned long)vmf->virtual_address,
1898 pfn + page_offset);
1899 }
Chris Wilsonc9839302012-11-20 10:45:17 +00001900unpin:
Joonas Lahtinenc5ad54c2015-05-06 14:36:09 +03001901 i915_gem_object_ggtt_unpin_view(obj, &view);
Chris Wilsonc7150892009-09-23 00:43:56 +01001902unlock:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001903 mutex_unlock(&dev->struct_mutex);
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001904out:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001905 switch (ret) {
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001906 case -EIO:
Daniel Vetter2232f032014-09-04 09:36:18 +02001907 /*
1908 * We eat errors when the gpu is terminally wedged to avoid
1909 * userspace unduly crashing (gl has no provisions for mmaps to
1910 * fail). But any other -EIO isn't ours (e.g. swap in failure)
1911 * and so needs to be reported.
1912 */
1913 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
Paulo Zanonif65c9162013-11-27 18:20:34 -02001914 ret = VM_FAULT_SIGBUS;
1915 break;
1916 }
Chris Wilson045e7692010-11-07 09:18:22 +00001917 case -EAGAIN:
Daniel Vetter571c6082013-09-12 17:57:28 +02001918 /*
1919 * EAGAIN means the gpu is hung and we'll wait for the error
1920 * handler to reset everything when re-faulting in
1921 * i915_mutex_lock_interruptible.
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001922 */
Chris Wilsonc7150892009-09-23 00:43:56 +01001923 case 0:
1924 case -ERESTARTSYS:
Chris Wilsonbed636a2011-02-11 20:31:19 +00001925 case -EINTR:
Dmitry Rogozhkine79e0fe2012-10-03 17:15:26 +03001926 case -EBUSY:
1927 /*
1928 * EBUSY is ok: this just means that another thread
1929 * already did the job.
1930 */
Paulo Zanonif65c9162013-11-27 18:20:34 -02001931 ret = VM_FAULT_NOPAGE;
1932 break;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001933 case -ENOMEM:
Paulo Zanonif65c9162013-11-27 18:20:34 -02001934 ret = VM_FAULT_OOM;
1935 break;
Daniel Vettera7c2e1a2012-10-17 11:17:16 +02001936 case -ENOSPC:
Chris Wilson45d67812014-01-31 11:34:57 +00001937 case -EFAULT:
Paulo Zanonif65c9162013-11-27 18:20:34 -02001938 ret = VM_FAULT_SIGBUS;
1939 break;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001940 default:
Daniel Vettera7c2e1a2012-10-17 11:17:16 +02001941 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
Paulo Zanonif65c9162013-11-27 18:20:34 -02001942 ret = VM_FAULT_SIGBUS;
1943 break;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001944 }
Paulo Zanonif65c9162013-11-27 18:20:34 -02001945
1946 intel_runtime_pm_put(dev_priv);
1947 return ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001948}
1949
1950/**
Chris Wilson901782b2009-07-10 08:18:50 +01001951 * i915_gem_release_mmap - remove physical page mappings
1952 * @obj: obj in question
1953 *
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02001954 * Preserve the reservation of the mmapping with the DRM core code, but
Chris Wilson901782b2009-07-10 08:18:50 +01001955 * relinquish ownership of the pages back to the system.
1956 *
1957 * It is vital that we remove the page mapping if we have mapped a tiled
1958 * object through the GTT and then lose the fence register due to
1959 * resource pressure. Similarly if the object has been moved out of the
1960 * aperture, than pages mapped into userspace must be revoked. Removing the
1961 * mapping will then trigger a page fault on the next user access, allowing
1962 * fixup by i915_gem_fault().
1963 */
Eric Anholtd05ca302009-07-10 13:02:26 -07001964void
Chris Wilson05394f32010-11-08 19:18:58 +00001965i915_gem_release_mmap(struct drm_i915_gem_object *obj)
Chris Wilson901782b2009-07-10 08:18:50 +01001966{
Chris Wilson6299f992010-11-24 12:23:44 +00001967 if (!obj->fault_mappable)
1968 return;
Chris Wilson901782b2009-07-10 08:18:50 +01001969
David Herrmann6796cb12014-01-03 14:24:19 +01001970 drm_vma_node_unmap(&obj->base.vma_node,
1971 obj->base.dev->anon_inode->i_mapping);
Chris Wilson6299f992010-11-24 12:23:44 +00001972 obj->fault_mappable = false;
Chris Wilson901782b2009-07-10 08:18:50 +01001973}
1974
Chris Wilsoneedd10f2014-06-16 08:57:44 +01001975void
1976i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1977{
1978 struct drm_i915_gem_object *obj;
1979
1980 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1981 i915_gem_release_mmap(obj);
1982}
1983
Imre Deak0fa87792013-01-07 21:47:35 +02001984uint32_t
Chris Wilsone28f8712011-07-18 13:11:49 -07001985i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
Chris Wilson92b88ae2010-11-09 11:47:32 +00001986{
Chris Wilsone28f8712011-07-18 13:11:49 -07001987 uint32_t gtt_size;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001988
1989 if (INTEL_INFO(dev)->gen >= 4 ||
Chris Wilsone28f8712011-07-18 13:11:49 -07001990 tiling_mode == I915_TILING_NONE)
1991 return size;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001992
1993 /* Previous chips need a power-of-two fence region when tiling */
1994 if (INTEL_INFO(dev)->gen == 3)
Chris Wilsone28f8712011-07-18 13:11:49 -07001995 gtt_size = 1024*1024;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001996 else
Chris Wilsone28f8712011-07-18 13:11:49 -07001997 gtt_size = 512*1024;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001998
Chris Wilsone28f8712011-07-18 13:11:49 -07001999 while (gtt_size < size)
2000 gtt_size <<= 1;
Chris Wilson92b88ae2010-11-09 11:47:32 +00002001
Chris Wilsone28f8712011-07-18 13:11:49 -07002002 return gtt_size;
Chris Wilson92b88ae2010-11-09 11:47:32 +00002003}
2004
Jesse Barnesde151cf2008-11-12 10:03:55 -08002005/**
2006 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
2007 * @obj: object to check
2008 *
2009 * Return the required GTT alignment for an object, taking into account
Daniel Vetter5e783302010-11-14 22:32:36 +01002010 * potential fence register mapping.
Jesse Barnesde151cf2008-11-12 10:03:55 -08002011 */
Imre Deakd8651102013-01-07 21:47:33 +02002012uint32_t
2013i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
2014 int tiling_mode, bool fenced)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002015{
Jesse Barnesde151cf2008-11-12 10:03:55 -08002016 /*
2017 * Minimum alignment is 4k (GTT page size), but might be greater
2018 * if a fence register is needed for the object.
2019 */
Imre Deakd8651102013-01-07 21:47:33 +02002020 if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
Chris Wilsone28f8712011-07-18 13:11:49 -07002021 tiling_mode == I915_TILING_NONE)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002022 return 4096;
2023
2024 /*
2025 * Previous chips need to be aligned to the size of the smallest
2026 * fence register that can contain the object.
2027 */
Chris Wilsone28f8712011-07-18 13:11:49 -07002028 return i915_gem_get_gtt_size(dev, size, tiling_mode);
Chris Wilsona00b10c2010-09-24 21:15:47 +01002029}
2030
Chris Wilsond8cb5082012-08-11 15:41:03 +01002031static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
2032{
2033 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2034 int ret;
2035
David Herrmann0de23972013-07-24 21:07:52 +02002036 if (drm_vma_node_has_offset(&obj->base.vma_node))
Chris Wilsond8cb5082012-08-11 15:41:03 +01002037 return 0;
2038
Daniel Vetterda494d72012-12-20 15:11:16 +01002039 dev_priv->mm.shrinker_no_lock_stealing = true;
2040
Chris Wilsond8cb5082012-08-11 15:41:03 +01002041 ret = drm_gem_create_mmap_offset(&obj->base);
2042 if (ret != -ENOSPC)
Daniel Vetterda494d72012-12-20 15:11:16 +01002043 goto out;
Chris Wilsond8cb5082012-08-11 15:41:03 +01002044
2045 /* Badly fragmented mmap space? The only way we can recover
2046 * space is by destroying unwanted objects. We can't randomly release
2047 * mmap_offsets as userspace expects them to be persistent for the
2048 * lifetime of the objects. The closest we can is to release the
2049 * offsets on purgeable objects by truncating it and marking it purged,
2050 * which prevents userspace from ever using that object again.
2051 */
Chris Wilson21ab4e72014-09-09 11:16:08 +01002052 i915_gem_shrink(dev_priv,
2053 obj->base.size >> PAGE_SHIFT,
2054 I915_SHRINK_BOUND |
2055 I915_SHRINK_UNBOUND |
2056 I915_SHRINK_PURGEABLE);
Chris Wilsond8cb5082012-08-11 15:41:03 +01002057 ret = drm_gem_create_mmap_offset(&obj->base);
2058 if (ret != -ENOSPC)
Daniel Vetterda494d72012-12-20 15:11:16 +01002059 goto out;
Chris Wilsond8cb5082012-08-11 15:41:03 +01002060
2061 i915_gem_shrink_all(dev_priv);
Daniel Vetterda494d72012-12-20 15:11:16 +01002062 ret = drm_gem_create_mmap_offset(&obj->base);
2063out:
2064 dev_priv->mm.shrinker_no_lock_stealing = false;
2065
2066 return ret;
Chris Wilsond8cb5082012-08-11 15:41:03 +01002067}
2068
2069static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2070{
Chris Wilsond8cb5082012-08-11 15:41:03 +01002071 drm_gem_free_mmap_offset(&obj->base);
2072}
2073
Dave Airlieda6b51d2014-12-24 13:11:17 +10002074int
Dave Airlieff72145b2011-02-07 12:16:14 +10002075i915_gem_mmap_gtt(struct drm_file *file,
2076 struct drm_device *dev,
Dave Airlieda6b51d2014-12-24 13:11:17 +10002077 uint32_t handle,
Dave Airlieff72145b2011-02-07 12:16:14 +10002078 uint64_t *offset)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002079{
Chris Wilson05394f32010-11-08 19:18:58 +00002080 struct drm_i915_gem_object *obj;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002081 int ret;
2082
Chris Wilson76c1dec2010-09-25 11:22:51 +01002083 ret = i915_mutex_lock_interruptible(dev);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01002084 if (ret)
Chris Wilson76c1dec2010-09-25 11:22:51 +01002085 return ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002086
Dave Airlieff72145b2011-02-07 12:16:14 +10002087 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00002088 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01002089 ret = -ENOENT;
2090 goto unlock;
2091 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08002092
Chris Wilson05394f32010-11-08 19:18:58 +00002093 if (obj->madv != I915_MADV_WILLNEED) {
Chris Wilsonbd9b6a42014-02-10 09:03:50 +00002094 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
Chris Wilson8c99e572014-01-31 11:34:58 +00002095 ret = -EFAULT;
Chris Wilson1d7cfea2010-10-17 09:45:41 +01002096 goto out;
Chris Wilsonab182822009-09-22 18:46:17 +01002097 }
2098
Chris Wilsond8cb5082012-08-11 15:41:03 +01002099 ret = i915_gem_object_create_mmap_offset(obj);
2100 if (ret)
2101 goto out;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002102
David Herrmann0de23972013-07-24 21:07:52 +02002103 *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002104
Chris Wilson1d7cfea2010-10-17 09:45:41 +01002105out:
Chris Wilson05394f32010-11-08 19:18:58 +00002106 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01002107unlock:
Jesse Barnesde151cf2008-11-12 10:03:55 -08002108 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01002109 return ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002110}
2111
Dave Airlieff72145b2011-02-07 12:16:14 +10002112/**
2113 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
2114 * @dev: DRM device
2115 * @data: GTT mapping ioctl data
2116 * @file: GEM object info
2117 *
2118 * Simply returns the fake offset to userspace so it can mmap it.
2119 * The mmap call will end up in drm_gem_mmap(), which will set things
2120 * up so we can get faults in the handler above.
2121 *
2122 * The fault handler will take care of binding the object into the GTT
2123 * (since it may have been evicted to make room for something), allocating
2124 * a fence register, and mapping the appropriate aperture address into
2125 * userspace.
2126 */
2127int
2128i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2129 struct drm_file *file)
2130{
2131 struct drm_i915_gem_mmap_gtt *args = data;
2132
Dave Airlieda6b51d2014-12-24 13:11:17 +10002133 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
Dave Airlieff72145b2011-02-07 12:16:14 +10002134}
2135
Daniel Vetter225067e2012-08-20 10:23:20 +02002136/* Immediately discard the backing storage */
2137static void
2138i915_gem_object_truncate(struct drm_i915_gem_object *obj)
Chris Wilsone5281cc2010-10-28 13:45:36 +01002139{
Chris Wilson4d6294bf2012-08-11 15:41:05 +01002140 i915_gem_object_free_mmap_offset(obj);
Daniel Vetter1286ff72012-05-10 15:25:09 +02002141
Chris Wilson4d6294bf2012-08-11 15:41:05 +01002142 if (obj->base.filp == NULL)
2143 return;
2144
Daniel Vetter225067e2012-08-20 10:23:20 +02002145 /* Our goal here is to return as much of the memory as
2146 * is possible back to the system as we are called from OOM.
2147 * To do this we must instruct the shmfs to drop all of its
2148 * backing pages, *now*.
Chris Wilsone5281cc2010-10-28 13:45:36 +01002149 */
Chris Wilson55372522014-03-25 13:23:06 +00002150 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
Daniel Vetter225067e2012-08-20 10:23:20 +02002151 obj->madv = __I915_MADV_PURGED;
Chris Wilsone5281cc2010-10-28 13:45:36 +01002152}
Chris Wilsone5281cc2010-10-28 13:45:36 +01002153
Chris Wilson55372522014-03-25 13:23:06 +00002154/* Try to discard unwanted pages */
2155static void
2156i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
Daniel Vetter225067e2012-08-20 10:23:20 +02002157{
Chris Wilson55372522014-03-25 13:23:06 +00002158 struct address_space *mapping;
2159
2160 switch (obj->madv) {
2161 case I915_MADV_DONTNEED:
2162 i915_gem_object_truncate(obj);
2163 case __I915_MADV_PURGED:
2164 return;
2165 }
2166
2167 if (obj->base.filp == NULL)
2168 return;
2169
2170 mapping = file_inode(obj->base.filp)->i_mapping,
2171 invalidate_mapping_pages(mapping, 0, (loff_t)-1);
Chris Wilsone5281cc2010-10-28 13:45:36 +01002172}
2173
Chris Wilson5cdf5882010-09-27 15:51:07 +01002174static void
Chris Wilson05394f32010-11-08 19:18:58 +00002175i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07002176{
Imre Deak90797e62013-02-18 19:28:03 +02002177 struct sg_page_iter sg_iter;
2178 int ret;
Daniel Vetter1286ff72012-05-10 15:25:09 +02002179
Chris Wilson05394f32010-11-08 19:18:58 +00002180 BUG_ON(obj->madv == __I915_MADV_PURGED);
Eric Anholt856fa192009-03-19 14:10:50 -07002181
Chris Wilson6c085a72012-08-20 11:40:46 +02002182 ret = i915_gem_object_set_to_cpu_domain(obj, true);
2183 if (ret) {
2184 /* In the event of a disaster, abandon all caches and
2185 * hope for the best.
2186 */
2187 WARN_ON(ret != -EIO);
Chris Wilson2c225692013-08-09 12:26:45 +01002188 i915_gem_clflush_object(obj, true);
Chris Wilson6c085a72012-08-20 11:40:46 +02002189 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2190 }
2191
Imre Deake2273302015-07-09 12:59:05 +03002192 i915_gem_gtt_finish_object(obj);
2193
Daniel Vetter6dacfd22011-09-12 21:30:02 +02002194 if (i915_gem_object_needs_bit17_swizzle(obj))
Eric Anholt280b7132009-03-12 16:56:27 -07002195 i915_gem_object_save_bit_17_swizzle(obj);
2196
Chris Wilson05394f32010-11-08 19:18:58 +00002197 if (obj->madv == I915_MADV_DONTNEED)
2198 obj->dirty = 0;
Chris Wilson3ef94da2009-09-14 16:50:29 +01002199
Imre Deak90797e62013-02-18 19:28:03 +02002200 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
Imre Deak2db76d72013-03-26 15:14:18 +02002201 struct page *page = sg_page_iter_page(&sg_iter);
Chris Wilson9da3da62012-06-01 15:20:22 +01002202
Chris Wilson05394f32010-11-08 19:18:58 +00002203 if (obj->dirty)
Chris Wilson9da3da62012-06-01 15:20:22 +01002204 set_page_dirty(page);
Chris Wilson3ef94da2009-09-14 16:50:29 +01002205
Chris Wilson05394f32010-11-08 19:18:58 +00002206 if (obj->madv == I915_MADV_WILLNEED)
Chris Wilson9da3da62012-06-01 15:20:22 +01002207 mark_page_accessed(page);
Chris Wilson3ef94da2009-09-14 16:50:29 +01002208
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002209 put_page(page);
Chris Wilson3ef94da2009-09-14 16:50:29 +01002210 }
Chris Wilson05394f32010-11-08 19:18:58 +00002211 obj->dirty = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07002212
Chris Wilson9da3da62012-06-01 15:20:22 +01002213 sg_free_table(obj->pages);
2214 kfree(obj->pages);
Chris Wilson37e680a2012-06-07 15:38:42 +01002215}
2216
Chris Wilsondd624af2013-01-15 12:39:35 +00002217int
Chris Wilson37e680a2012-06-07 15:38:42 +01002218i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2219{
2220 const struct drm_i915_gem_object_ops *ops = obj->ops;
2221
Chris Wilson2f745ad2012-09-04 21:02:58 +01002222 if (obj->pages == NULL)
Chris Wilson37e680a2012-06-07 15:38:42 +01002223 return 0;
2224
Chris Wilsona5570172012-09-04 21:02:54 +01002225 if (obj->pages_pin_count)
2226 return -EBUSY;
2227
Ben Widawsky98438772013-07-31 17:00:12 -07002228 BUG_ON(i915_gem_obj_bound_any(obj));
Ben Widawsky3e123022013-07-31 17:00:04 -07002229
Chris Wilsona2165e32012-12-03 11:49:00 +00002230 /* ->put_pages might need to allocate memory for the bit17 swizzle
2231 * array, hence protect them from being reaped by removing them from gtt
2232 * lists early. */
Ben Widawsky35c20a62013-05-31 11:28:48 -07002233 list_del(&obj->global_list);
Chris Wilsona2165e32012-12-03 11:49:00 +00002234
Chris Wilson0a798eb2016-04-08 12:11:11 +01002235 if (obj->mapping) {
Chris Wilsonfb8621d2016-04-08 12:11:14 +01002236 if (is_vmalloc_addr(obj->mapping))
2237 vunmap(obj->mapping);
2238 else
2239 kunmap(kmap_to_page(obj->mapping));
Chris Wilson0a798eb2016-04-08 12:11:11 +01002240 obj->mapping = NULL;
2241 }
2242
Chris Wilson37e680a2012-06-07 15:38:42 +01002243 ops->put_pages(obj);
Chris Wilson05394f32010-11-08 19:18:58 +00002244 obj->pages = NULL;
Chris Wilson6c085a72012-08-20 11:40:46 +02002245
Chris Wilson55372522014-03-25 13:23:06 +00002246 i915_gem_object_invalidate(obj);
Chris Wilson6c085a72012-08-20 11:40:46 +02002247
2248 return 0;
2249}
2250
Chris Wilson37e680a2012-06-07 15:38:42 +01002251static int
Chris Wilson6c085a72012-08-20 11:40:46 +02002252i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07002253{
Chris Wilson6c085a72012-08-20 11:40:46 +02002254 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07002255 int page_count, i;
2256 struct address_space *mapping;
Chris Wilson9da3da62012-06-01 15:20:22 +01002257 struct sg_table *st;
2258 struct scatterlist *sg;
Imre Deak90797e62013-02-18 19:28:03 +02002259 struct sg_page_iter sg_iter;
Eric Anholt673a3942008-07-30 12:06:12 -07002260 struct page *page;
Imre Deak90797e62013-02-18 19:28:03 +02002261 unsigned long last_pfn = 0; /* suppress gcc warning */
Imre Deake2273302015-07-09 12:59:05 +03002262 int ret;
Chris Wilson6c085a72012-08-20 11:40:46 +02002263 gfp_t gfp;
Eric Anholt673a3942008-07-30 12:06:12 -07002264
Chris Wilson6c085a72012-08-20 11:40:46 +02002265 /* Assert that the object is not currently in any GPU domain. As it
2266 * wasn't in the GTT, there shouldn't be any way it could have been in
2267 * a GPU cache
2268 */
2269 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2270 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2271
Chris Wilson9da3da62012-06-01 15:20:22 +01002272 st = kmalloc(sizeof(*st), GFP_KERNEL);
2273 if (st == NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07002274 return -ENOMEM;
2275
Chris Wilson9da3da62012-06-01 15:20:22 +01002276 page_count = obj->base.size / PAGE_SIZE;
2277 if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
Chris Wilson9da3da62012-06-01 15:20:22 +01002278 kfree(st);
2279 return -ENOMEM;
2280 }
2281
2282 /* Get the list of pages out of our struct file. They'll be pinned
2283 * at this point until we release them.
2284 *
2285 * Fail silently without starting the shrinker
2286 */
Al Viro496ad9a2013-01-23 17:07:38 -05002287 mapping = file_inode(obj->base.filp)->i_mapping;
Michal Hockoc62d2552015-11-06 16:28:49 -08002288 gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
Mel Gormand0164ad2015-11-06 16:28:21 -08002289 gfp |= __GFP_NORETRY | __GFP_NOWARN;
Imre Deak90797e62013-02-18 19:28:03 +02002290 sg = st->sgl;
2291 st->nents = 0;
2292 for (i = 0; i < page_count; i++) {
Chris Wilson6c085a72012-08-20 11:40:46 +02002293 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2294 if (IS_ERR(page)) {
Chris Wilson21ab4e72014-09-09 11:16:08 +01002295 i915_gem_shrink(dev_priv,
2296 page_count,
2297 I915_SHRINK_BOUND |
2298 I915_SHRINK_UNBOUND |
2299 I915_SHRINK_PURGEABLE);
Chris Wilson6c085a72012-08-20 11:40:46 +02002300 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2301 }
2302 if (IS_ERR(page)) {
2303 /* We've tried hard to allocate the memory by reaping
2304 * our own buffer, now let the real VM do its job and
2305 * go down in flames if truly OOM.
2306 */
Chris Wilson6c085a72012-08-20 11:40:46 +02002307 i915_gem_shrink_all(dev_priv);
David Herrmannf461d1b2014-05-25 14:34:10 +02002308 page = shmem_read_mapping_page(mapping, i);
Imre Deake2273302015-07-09 12:59:05 +03002309 if (IS_ERR(page)) {
2310 ret = PTR_ERR(page);
Chris Wilson6c085a72012-08-20 11:40:46 +02002311 goto err_pages;
Imre Deake2273302015-07-09 12:59:05 +03002312 }
Chris Wilson6c085a72012-08-20 11:40:46 +02002313 }
Konrad Rzeszutek Wilk426729d2013-06-24 11:47:48 -04002314#ifdef CONFIG_SWIOTLB
2315 if (swiotlb_nr_tbl()) {
2316 st->nents++;
2317 sg_set_page(sg, page, PAGE_SIZE, 0);
2318 sg = sg_next(sg);
2319 continue;
2320 }
2321#endif
Imre Deak90797e62013-02-18 19:28:03 +02002322 if (!i || page_to_pfn(page) != last_pfn + 1) {
2323 if (i)
2324 sg = sg_next(sg);
2325 st->nents++;
2326 sg_set_page(sg, page, PAGE_SIZE, 0);
2327 } else {
2328 sg->length += PAGE_SIZE;
2329 }
2330 last_pfn = page_to_pfn(page);
Daniel Vetter3bbbe702013-10-07 17:15:45 -03002331
2332 /* Check that the i965g/gm workaround works. */
2333 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
Eric Anholt673a3942008-07-30 12:06:12 -07002334 }
Konrad Rzeszutek Wilk426729d2013-06-24 11:47:48 -04002335#ifdef CONFIG_SWIOTLB
2336 if (!swiotlb_nr_tbl())
2337#endif
2338 sg_mark_end(sg);
Chris Wilson74ce6b62012-10-19 15:51:06 +01002339 obj->pages = st;
2340
Imre Deake2273302015-07-09 12:59:05 +03002341 ret = i915_gem_gtt_prepare_object(obj);
2342 if (ret)
2343 goto err_pages;
2344
Eric Anholt673a3942008-07-30 12:06:12 -07002345 if (i915_gem_object_needs_bit17_swizzle(obj))
2346 i915_gem_object_do_bit_17_swizzle(obj);
2347
Daniel Vetter656bfa32014-11-20 09:26:30 +01002348 if (obj->tiling_mode != I915_TILING_NONE &&
2349 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2350 i915_gem_object_pin_pages(obj);
2351
Eric Anholt673a3942008-07-30 12:06:12 -07002352 return 0;
2353
2354err_pages:
Imre Deak90797e62013-02-18 19:28:03 +02002355 sg_mark_end(sg);
2356 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002357 put_page(sg_page_iter_page(&sg_iter));
Chris Wilson9da3da62012-06-01 15:20:22 +01002358 sg_free_table(st);
2359 kfree(st);
Chris Wilson0820baf2014-03-25 13:23:03 +00002360
2361 /* shmemfs first checks if there is enough memory to allocate the page
2362 * and reports ENOSPC should there be insufficient, along with the usual
2363 * ENOMEM for a genuine allocation failure.
2364 *
2365 * We use ENOSPC in our driver to mean that we have run out of aperture
2366 * space and so want to translate the error from shmemfs back to our
2367 * usual understanding of ENOMEM.
2368 */
Imre Deake2273302015-07-09 12:59:05 +03002369 if (ret == -ENOSPC)
2370 ret = -ENOMEM;
2371
2372 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002373}
2374
Chris Wilson37e680a2012-06-07 15:38:42 +01002375/* Ensure that the associated pages are gathered from the backing storage
2376 * and pinned into our object. i915_gem_object_get_pages() may be called
2377 * multiple times before they are released by a single call to
2378 * i915_gem_object_put_pages() - once the pages are no longer referenced
2379 * either as a result of memory pressure (reaping pages under the shrinker)
2380 * or as the object is itself released.
2381 */
2382int
2383i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2384{
2385 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2386 const struct drm_i915_gem_object_ops *ops = obj->ops;
2387 int ret;
2388
Chris Wilson2f745ad2012-09-04 21:02:58 +01002389 if (obj->pages)
Chris Wilson37e680a2012-06-07 15:38:42 +01002390 return 0;
2391
Chris Wilson43e28f02013-01-08 10:53:09 +00002392 if (obj->madv != I915_MADV_WILLNEED) {
Chris Wilsonbd9b6a42014-02-10 09:03:50 +00002393 DRM_DEBUG("Attempting to obtain a purgeable object\n");
Chris Wilson8c99e572014-01-31 11:34:58 +00002394 return -EFAULT;
Chris Wilson43e28f02013-01-08 10:53:09 +00002395 }
2396
Chris Wilsona5570172012-09-04 21:02:54 +01002397 BUG_ON(obj->pages_pin_count);
2398
Chris Wilson37e680a2012-06-07 15:38:42 +01002399 ret = ops->get_pages(obj);
2400 if (ret)
2401 return ret;
2402
Ben Widawsky35c20a62013-05-31 11:28:48 -07002403 list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
Chris Wilsonee286372015-04-07 16:20:25 +01002404
2405 obj->get_page.sg = obj->pages->sgl;
2406 obj->get_page.last = 0;
2407
Chris Wilson37e680a2012-06-07 15:38:42 +01002408 return 0;
Eric Anholt673a3942008-07-30 12:06:12 -07002409}
2410
Chris Wilson0a798eb2016-04-08 12:11:11 +01002411void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj)
2412{
2413 int ret;
2414
2415 lockdep_assert_held(&obj->base.dev->struct_mutex);
2416
2417 ret = i915_gem_object_get_pages(obj);
2418 if (ret)
2419 return ERR_PTR(ret);
2420
2421 i915_gem_object_pin_pages(obj);
2422
2423 if (obj->mapping == NULL) {
Chris Wilson0a798eb2016-04-08 12:11:11 +01002424 struct page **pages;
Chris Wilson0a798eb2016-04-08 12:11:11 +01002425
Chris Wilsonfb8621d2016-04-08 12:11:14 +01002426 pages = NULL;
2427 if (obj->base.size == PAGE_SIZE)
2428 obj->mapping = kmap(sg_page(obj->pages->sgl));
2429 else
2430 pages = drm_malloc_gfp(obj->base.size >> PAGE_SHIFT,
2431 sizeof(*pages),
2432 GFP_TEMPORARY);
Chris Wilson0a798eb2016-04-08 12:11:11 +01002433 if (pages != NULL) {
Chris Wilsonfb8621d2016-04-08 12:11:14 +01002434 struct sg_page_iter sg_iter;
2435 int n;
2436
Chris Wilson0a798eb2016-04-08 12:11:11 +01002437 n = 0;
2438 for_each_sg_page(obj->pages->sgl, &sg_iter,
2439 obj->pages->nents, 0)
2440 pages[n++] = sg_page_iter_page(&sg_iter);
2441
2442 obj->mapping = vmap(pages, n, 0, PAGE_KERNEL);
2443 drm_free_large(pages);
2444 }
2445 if (obj->mapping == NULL) {
2446 i915_gem_object_unpin_pages(obj);
2447 return ERR_PTR(-ENOMEM);
2448 }
2449 }
2450
2451 return obj->mapping;
2452}
2453
Ben Widawskye2d05a82013-09-24 09:57:58 -07002454void i915_vma_move_to_active(struct i915_vma *vma,
John Harrisonb2af0372015-05-29 17:43:50 +01002455 struct drm_i915_gem_request *req)
Ben Widawskye2d05a82013-09-24 09:57:58 -07002456{
Chris Wilsonb4716182015-04-27 13:41:17 +01002457 struct drm_i915_gem_object *obj = vma->obj;
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00002458 struct intel_engine_cs *engine;
John Harrisonb2af0372015-05-29 17:43:50 +01002459
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00002460 engine = i915_gem_request_get_engine(req);
Chris Wilsonb4716182015-04-27 13:41:17 +01002461
2462 /* Add a reference if we're newly entering the active list. */
2463 if (obj->active == 0)
2464 drm_gem_object_reference(&obj->base);
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00002465 obj->active |= intel_engine_flag(engine);
Chris Wilsonb4716182015-04-27 13:41:17 +01002466
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00002467 list_move_tail(&obj->engine_list[engine->id], &engine->active_list);
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00002468 i915_gem_request_assign(&obj->last_read_req[engine->id], req);
Chris Wilsonb4716182015-04-27 13:41:17 +01002469
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00002470 list_move_tail(&vma->vm_link, &vma->vm->active_list);
Ben Widawskye2d05a82013-09-24 09:57:58 -07002471}
2472
Chris Wilsoncaea7472010-11-12 13:53:37 +00002473static void
Chris Wilsonb4716182015-04-27 13:41:17 +01002474i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
2475{
2476 RQ_BUG_ON(obj->last_write_req == NULL);
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00002477 RQ_BUG_ON(!(obj->active & intel_engine_flag(obj->last_write_req->engine)));
Chris Wilsonb4716182015-04-27 13:41:17 +01002478
2479 i915_gem_request_assign(&obj->last_write_req, NULL);
Rodrigo Vivide152b62015-07-07 16:28:51 -07002480 intel_fb_obj_flush(obj, true, ORIGIN_CS);
Chris Wilsonb4716182015-04-27 13:41:17 +01002481}
2482
2483static void
2484i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
Chris Wilsoncaea7472010-11-12 13:53:37 +00002485{
Ben Widawskyfeb822c2013-12-06 14:10:51 -08002486 struct i915_vma *vma;
Chris Wilsoncaea7472010-11-12 13:53:37 +00002487
Chris Wilsonb4716182015-04-27 13:41:17 +01002488 RQ_BUG_ON(obj->last_read_req[ring] == NULL);
2489 RQ_BUG_ON(!(obj->active & (1 << ring)));
2490
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00002491 list_del_init(&obj->engine_list[ring]);
Chris Wilsonb4716182015-04-27 13:41:17 +01002492 i915_gem_request_assign(&obj->last_read_req[ring], NULL);
2493
Tvrtko Ursulin4a570db2016-03-16 11:00:38 +00002494 if (obj->last_write_req && obj->last_write_req->engine->id == ring)
Chris Wilsonb4716182015-04-27 13:41:17 +01002495 i915_gem_object_retire__write(obj);
2496
2497 obj->active &= ~(1 << ring);
2498 if (obj->active)
2499 return;
Chris Wilson65ce3022012-07-20 12:41:02 +01002500
Chris Wilson6c246952015-07-27 10:26:26 +01002501 /* Bump our place on the bound list to keep it roughly in LRU order
2502 * so that we don't steal from recently used but inactive objects
2503 * (unless we are forced to ofc!)
2504 */
2505 list_move_tail(&obj->global_list,
2506 &to_i915(obj->base.dev)->mm.bound_list);
2507
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00002508 list_for_each_entry(vma, &obj->vma_list, obj_link) {
2509 if (!list_empty(&vma->vm_link))
2510 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
Ben Widawskyfeb822c2013-12-06 14:10:51 -08002511 }
Chris Wilsoncaea7472010-11-12 13:53:37 +00002512
John Harrison97b2a6a2014-11-24 18:49:26 +00002513 i915_gem_request_assign(&obj->last_fenced_req, NULL);
Chris Wilsoncaea7472010-11-12 13:53:37 +00002514 drm_gem_object_unreference(&obj->base);
Chris Wilsonc8725f32014-03-17 12:21:55 +00002515}
2516
Chris Wilson9d7730912012-11-27 16:22:52 +00002517static int
Mika Kuoppalafca26bb2012-12-19 11:13:08 +02002518i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
Daniel Vetter53d227f2012-01-25 16:32:49 +01002519{
Chris Wilson9d7730912012-11-27 16:22:52 +00002520 struct drm_i915_private *dev_priv = dev->dev_private;
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00002521 struct intel_engine_cs *engine;
Chris Wilson29dcb572016-04-07 07:29:13 +01002522 int ret;
Daniel Vetter53d227f2012-01-25 16:32:49 +01002523
Chris Wilson107f27a52012-12-10 13:56:17 +02002524 /* Carefully retire all requests without writing to the rings */
Dave Gordonb4ac5af2016-03-24 11:20:38 +00002525 for_each_engine(engine, dev_priv) {
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00002526 ret = intel_engine_idle(engine);
Chris Wilson107f27a52012-12-10 13:56:17 +02002527 if (ret)
2528 return ret;
Chris Wilson9d7730912012-11-27 16:22:52 +00002529 }
Chris Wilson9d7730912012-11-27 16:22:52 +00002530 i915_gem_retire_requests(dev);
Chris Wilson107f27a52012-12-10 13:56:17 +02002531
2532 /* Finally reset hw state */
Chris Wilson29dcb572016-04-07 07:29:13 +01002533 for_each_engine(engine, dev_priv)
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00002534 intel_ring_init_seqno(engine, seqno);
Mika Kuoppala498d2ac2012-12-04 15:12:04 +02002535
Chris Wilson9d7730912012-11-27 16:22:52 +00002536 return 0;
Daniel Vetter53d227f2012-01-25 16:32:49 +01002537}
2538
Mika Kuoppalafca26bb2012-12-19 11:13:08 +02002539int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2540{
2541 struct drm_i915_private *dev_priv = dev->dev_private;
2542 int ret;
2543
2544 if (seqno == 0)
2545 return -EINVAL;
2546
2547 /* HWS page needs to be set less than what we
2548 * will inject to ring
2549 */
2550 ret = i915_gem_init_seqno(dev, seqno - 1);
2551 if (ret)
2552 return ret;
2553
2554 /* Carefully set the last_seqno value so that wrap
2555 * detection still works
2556 */
2557 dev_priv->next_seqno = seqno;
2558 dev_priv->last_seqno = seqno - 1;
2559 if (dev_priv->last_seqno == 0)
2560 dev_priv->last_seqno--;
2561
2562 return 0;
2563}
2564
Chris Wilson9d7730912012-11-27 16:22:52 +00002565int
2566i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
Daniel Vetter53d227f2012-01-25 16:32:49 +01002567{
Chris Wilson9d7730912012-11-27 16:22:52 +00002568 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter53d227f2012-01-25 16:32:49 +01002569
Chris Wilson9d7730912012-11-27 16:22:52 +00002570 /* reserve 0 for non-seqno */
2571 if (dev_priv->next_seqno == 0) {
Mika Kuoppalafca26bb2012-12-19 11:13:08 +02002572 int ret = i915_gem_init_seqno(dev, 0);
Chris Wilson9d7730912012-11-27 16:22:52 +00002573 if (ret)
2574 return ret;
2575
2576 dev_priv->next_seqno = 1;
2577 }
2578
Mika Kuoppalaf72b3432012-12-10 15:41:48 +02002579 *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
Chris Wilson9d7730912012-11-27 16:22:52 +00002580 return 0;
Daniel Vetter53d227f2012-01-25 16:32:49 +01002581}
2582
John Harrisonbf7dc5b2015-05-29 17:43:24 +01002583/*
2584 * NB: This function is not allowed to fail. Doing so would mean the the
2585 * request is not being tracked for completion but the work itself is
2586 * going to happen on the hardware. This would be a Bad Thing(tm).
2587 */
John Harrison75289872015-05-29 17:43:49 +01002588void __i915_add_request(struct drm_i915_gem_request *request,
John Harrison5b4a60c2015-05-29 17:43:34 +01002589 struct drm_i915_gem_object *obj,
2590 bool flush_caches)
Eric Anholt673a3942008-07-30 12:06:12 -07002591{
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00002592 struct intel_engine_cs *engine;
John Harrison75289872015-05-29 17:43:49 +01002593 struct drm_i915_private *dev_priv;
Oscar Mateo48e29f52014-07-24 17:04:29 +01002594 struct intel_ringbuffer *ringbuf;
Nick Hoath6d3d8272015-01-15 13:10:39 +00002595 u32 request_start;
Chris Wilson3cce4692010-10-27 16:11:02 +01002596 int ret;
2597
Oscar Mateo48e29f52014-07-24 17:04:29 +01002598 if (WARN_ON(request == NULL))
John Harrisonbf7dc5b2015-05-29 17:43:24 +01002599 return;
Oscar Mateo48e29f52014-07-24 17:04:29 +01002600
Tvrtko Ursulin4a570db2016-03-16 11:00:38 +00002601 engine = request->engine;
Tvrtko Ursulin39dabec2016-03-17 13:04:10 +00002602 dev_priv = request->i915;
John Harrison75289872015-05-29 17:43:49 +01002603 ringbuf = request->ringbuf;
2604
John Harrison29b1b412015-06-18 13:10:09 +01002605 /*
2606 * To ensure that this call will not fail, space for its emissions
2607 * should already have been reserved in the ring buffer. Let the ring
2608 * know that it is time to use that space up.
2609 */
2610 intel_ring_reserved_space_use(ringbuf);
2611
Oscar Mateo48e29f52014-07-24 17:04:29 +01002612 request_start = intel_ring_get_tail(ringbuf);
Daniel Vettercc889e02012-06-13 20:45:19 +02002613 /*
2614 * Emit any outstanding flushes - execbuf can fail to emit the flush
2615 * after having emitted the batchbuffer command. Hence we need to fix
2616 * things up similar to emitting the lazy request. The difference here
2617 * is that the flush _must_ happen before the next request, no matter
2618 * what.
2619 */
John Harrison5b4a60c2015-05-29 17:43:34 +01002620 if (flush_caches) {
2621 if (i915.enable_execlists)
John Harrison4866d722015-05-29 17:43:55 +01002622 ret = logical_ring_flush_all_caches(request);
John Harrison5b4a60c2015-05-29 17:43:34 +01002623 else
John Harrison4866d722015-05-29 17:43:55 +01002624 ret = intel_ring_flush_all_caches(request);
John Harrison5b4a60c2015-05-29 17:43:34 +01002625 /* Not allowed to fail! */
2626 WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
2627 }
Daniel Vettercc889e02012-06-13 20:45:19 +02002628
Chris Wilson7c90b7d2016-04-07 07:29:17 +01002629 trace_i915_gem_request_add(request);
2630
2631 request->head = request_start;
2632
2633 /* Whilst this request exists, batch_obj will be on the
2634 * active_list, and so will hold the active reference. Only when this
2635 * request is retired will the the batch_obj be moved onto the
2636 * inactive_list and lose its active reference. Hence we do not need
2637 * to explicitly hold another reference here.
2638 */
2639 request->batch_obj = obj;
2640
2641 /* Seal the request and mark it as pending execution. Note that
2642 * we may inspect this state, without holding any locks, during
2643 * hangcheck. Hence we apply the barrier to ensure that we do not
2644 * see a more recent value in the hws than we are tracking.
2645 */
2646 request->emitted_jiffies = jiffies;
2647 request->previous_seqno = engine->last_submitted_seqno;
2648 smp_store_mb(engine->last_submitted_seqno, request->seqno);
2649 list_add_tail(&request->list, &engine->request_list);
2650
Chris Wilsona71d8d92012-02-15 11:25:36 +00002651 /* Record the position of the start of the request so that
2652 * should we detect the updated seqno part-way through the
2653 * GPU processing the request, we never over-estimate the
2654 * position of the head.
2655 */
Nick Hoath6d3d8272015-01-15 13:10:39 +00002656 request->postfix = intel_ring_get_tail(ringbuf);
Chris Wilsona71d8d92012-02-15 11:25:36 +00002657
John Harrisonbf7dc5b2015-05-29 17:43:24 +01002658 if (i915.enable_execlists)
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00002659 ret = engine->emit_request(request);
John Harrisonbf7dc5b2015-05-29 17:43:24 +01002660 else {
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00002661 ret = engine->add_request(request);
Michel Thierry53292cd2015-04-15 18:11:33 +01002662
2663 request->tail = intel_ring_get_tail(ringbuf);
Oscar Mateo48e29f52014-07-24 17:04:29 +01002664 }
John Harrisonbf7dc5b2015-05-29 17:43:24 +01002665 /* Not allowed to fail! */
2666 WARN(ret, "emit|add_request failed: %d!\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07002667
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00002668 i915_queue_hangcheck(engine->dev);
Mika Kuoppala10cd45b2013-07-03 17:22:08 +03002669
Daniel Vetter87255482014-11-19 20:36:48 +01002670 queue_delayed_work(dev_priv->wq,
2671 &dev_priv->mm.retire_work,
2672 round_jiffies_up_relative(HZ));
2673 intel_mark_busy(dev_priv->dev);
Daniel Vettercc889e02012-06-13 20:45:19 +02002674
John Harrison29b1b412015-06-18 13:10:09 +01002675 /* Sanity check that the reserved size was large enough. */
2676 intel_ring_reserved_space_end(ringbuf);
Eric Anholt673a3942008-07-30 12:06:12 -07002677}
2678
Mika Kuoppala939fd762014-01-30 19:04:44 +02002679static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
Oscar Mateo273497e2014-05-22 14:13:37 +01002680 const struct intel_context *ctx)
Mika Kuoppalabe62acb2013-08-30 16:19:28 +03002681{
Mika Kuoppala44e2c072014-01-30 16:01:15 +02002682 unsigned long elapsed;
Mika Kuoppalabe62acb2013-08-30 16:19:28 +03002683
Mika Kuoppala44e2c072014-01-30 16:01:15 +02002684 elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2685
2686 if (ctx->hang_stats.banned)
Mika Kuoppalabe62acb2013-08-30 16:19:28 +03002687 return true;
2688
Chris Wilson676fa572014-12-24 08:13:39 -08002689 if (ctx->hang_stats.ban_period_seconds &&
2690 elapsed <= ctx->hang_stats.ban_period_seconds) {
Ville Syrjäläccc7bed2014-02-21 16:26:47 +02002691 if (!i915_gem_context_is_default(ctx)) {
Mika Kuoppala3fac8972014-01-30 16:05:48 +02002692 DRM_DEBUG("context hanging too fast, banning!\n");
Ville Syrjäläccc7bed2014-02-21 16:26:47 +02002693 return true;
Mika Kuoppala88b4aa82014-03-28 18:18:18 +02002694 } else if (i915_stop_ring_allow_ban(dev_priv)) {
2695 if (i915_stop_ring_allow_warn(dev_priv))
2696 DRM_ERROR("gpu hanging too fast, banning!\n");
Ville Syrjäläccc7bed2014-02-21 16:26:47 +02002697 return true;
Mika Kuoppala3fac8972014-01-30 16:05:48 +02002698 }
Mika Kuoppalabe62acb2013-08-30 16:19:28 +03002699 }
2700
2701 return false;
2702}
2703
Mika Kuoppala939fd762014-01-30 19:04:44 +02002704static void i915_set_reset_status(struct drm_i915_private *dev_priv,
Oscar Mateo273497e2014-05-22 14:13:37 +01002705 struct intel_context *ctx,
Mika Kuoppalab6b0fac2014-01-30 19:04:43 +02002706 const bool guilty)
Mika Kuoppalaaa60c662013-06-12 15:13:20 +03002707{
Mika Kuoppala44e2c072014-01-30 16:01:15 +02002708 struct i915_ctx_hang_stats *hs;
2709
2710 if (WARN_ON(!ctx))
2711 return;
Mika Kuoppalaaa60c662013-06-12 15:13:20 +03002712
Mika Kuoppala44e2c072014-01-30 16:01:15 +02002713 hs = &ctx->hang_stats;
2714
2715 if (guilty) {
Mika Kuoppala939fd762014-01-30 19:04:44 +02002716 hs->banned = i915_context_is_banned(dev_priv, ctx);
Mika Kuoppala44e2c072014-01-30 16:01:15 +02002717 hs->batch_active++;
2718 hs->guilty_ts = get_seconds();
2719 } else {
2720 hs->batch_pending++;
Mika Kuoppalaaa60c662013-06-12 15:13:20 +03002721 }
2722}
2723
John Harrisonabfe2622014-11-24 18:49:24 +00002724void i915_gem_request_free(struct kref *req_ref)
2725{
2726 struct drm_i915_gem_request *req = container_of(req_ref,
2727 typeof(*req), ref);
2728 struct intel_context *ctx = req->ctx;
2729
John Harrisonfcfa423c2015-05-29 17:44:12 +01002730 if (req->file_priv)
2731 i915_gem_request_remove_from_client(req);
2732
Thomas Daniel0794aed2014-11-25 10:39:25 +00002733 if (ctx) {
Dave Gordone28e4042016-01-19 19:02:55 +00002734 if (i915.enable_execlists && ctx != req->i915->kernel_context)
Tvrtko Ursulin4a570db2016-03-16 11:00:38 +00002735 intel_lr_context_unpin(ctx, req->engine);
John Harrisonabfe2622014-11-24 18:49:24 +00002736
Oscar Mateodcb4c122014-11-13 10:28:10 +00002737 i915_gem_context_unreference(ctx);
2738 }
John Harrisonabfe2622014-11-24 18:49:24 +00002739
Chris Wilsonefab6d82015-04-07 16:20:57 +01002740 kmem_cache_free(req->i915->requests, req);
Mika Kuoppala0e50e962013-05-02 16:48:08 +03002741}
2742
Dave Gordon26827082016-01-19 19:02:53 +00002743static inline int
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002744__i915_gem_request_alloc(struct intel_engine_cs *engine,
Dave Gordon26827082016-01-19 19:02:53 +00002745 struct intel_context *ctx,
2746 struct drm_i915_gem_request **req_out)
John Harrison6689cb22015-03-19 12:30:08 +00002747{
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002748 struct drm_i915_private *dev_priv = to_i915(engine->dev);
Daniel Vettereed29a52015-05-21 14:21:25 +02002749 struct drm_i915_gem_request *req;
John Harrison6689cb22015-03-19 12:30:08 +00002750 int ret;
John Harrison6689cb22015-03-19 12:30:08 +00002751
John Harrison217e46b2015-05-29 17:43:29 +01002752 if (!req_out)
2753 return -EINVAL;
2754
John Harrisonbccca492015-05-29 17:44:11 +01002755 *req_out = NULL;
John Harrison6689cb22015-03-19 12:30:08 +00002756
Daniel Vettereed29a52015-05-21 14:21:25 +02002757 req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
2758 if (req == NULL)
John Harrison6689cb22015-03-19 12:30:08 +00002759 return -ENOMEM;
2760
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002761 ret = i915_gem_get_seqno(engine->dev, &req->seqno);
Chris Wilson9a0c1e22015-05-21 21:01:45 +01002762 if (ret)
2763 goto err;
John Harrison6689cb22015-03-19 12:30:08 +00002764
John Harrison40e895c2015-05-29 17:43:26 +01002765 kref_init(&req->ref);
2766 req->i915 = dev_priv;
Tvrtko Ursulin4a570db2016-03-16 11:00:38 +00002767 req->engine = engine;
John Harrison40e895c2015-05-29 17:43:26 +01002768 req->ctx = ctx;
2769 i915_gem_context_reference(req->ctx);
John Harrison6689cb22015-03-19 12:30:08 +00002770
2771 if (i915.enable_execlists)
John Harrison40e895c2015-05-29 17:43:26 +01002772 ret = intel_logical_ring_alloc_request_extras(req);
John Harrison6689cb22015-03-19 12:30:08 +00002773 else
Daniel Vettereed29a52015-05-21 14:21:25 +02002774 ret = intel_ring_alloc_request_extras(req);
John Harrison40e895c2015-05-29 17:43:26 +01002775 if (ret) {
2776 i915_gem_context_unreference(req->ctx);
Chris Wilson9a0c1e22015-05-21 21:01:45 +01002777 goto err;
John Harrison40e895c2015-05-29 17:43:26 +01002778 }
John Harrison6689cb22015-03-19 12:30:08 +00002779
John Harrison29b1b412015-06-18 13:10:09 +01002780 /*
2781 * Reserve space in the ring buffer for all the commands required to
2782 * eventually emit this request. This is to guarantee that the
2783 * i915_add_request() call can't fail. Note that the reserve may need
2784 * to be redone if the request is not actually submitted straight
2785 * away, e.g. because a GPU scheduler has deferred it.
John Harrison29b1b412015-06-18 13:10:09 +01002786 */
John Harrisonccd98fe2015-05-29 17:44:09 +01002787 if (i915.enable_execlists)
2788 ret = intel_logical_ring_reserve_space(req);
2789 else
2790 ret = intel_ring_reserve_space(req);
2791 if (ret) {
2792 /*
2793 * At this point, the request is fully allocated even if not
2794 * fully prepared. Thus it can be cleaned up using the proper
2795 * free code.
2796 */
2797 i915_gem_request_cancel(req);
2798 return ret;
2799 }
John Harrison29b1b412015-06-18 13:10:09 +01002800
John Harrisonbccca492015-05-29 17:44:11 +01002801 *req_out = req;
John Harrison6689cb22015-03-19 12:30:08 +00002802 return 0;
Chris Wilson9a0c1e22015-05-21 21:01:45 +01002803
2804err:
2805 kmem_cache_free(dev_priv->requests, req);
2806 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002807}
2808
Dave Gordon26827082016-01-19 19:02:53 +00002809/**
2810 * i915_gem_request_alloc - allocate a request structure
2811 *
2812 * @engine: engine that we wish to issue the request on.
2813 * @ctx: context that the request will be associated with.
2814 * This can be NULL if the request is not directly related to
2815 * any specific user context, in which case this function will
2816 * choose an appropriate context to use.
2817 *
2818 * Returns a pointer to the allocated request if successful,
2819 * or an error code if not.
2820 */
2821struct drm_i915_gem_request *
2822i915_gem_request_alloc(struct intel_engine_cs *engine,
2823 struct intel_context *ctx)
2824{
2825 struct drm_i915_gem_request *req;
2826 int err;
2827
2828 if (ctx == NULL)
Dave Gordoned54c1a2016-01-19 19:02:54 +00002829 ctx = to_i915(engine->dev)->kernel_context;
Dave Gordon26827082016-01-19 19:02:53 +00002830 err = __i915_gem_request_alloc(engine, ctx, &req);
2831 return err ? ERR_PTR(err) : req;
2832}
2833
John Harrison29b1b412015-06-18 13:10:09 +01002834void i915_gem_request_cancel(struct drm_i915_gem_request *req)
2835{
2836 intel_ring_reserved_space_cancel(req->ringbuf);
2837
2838 i915_gem_request_unreference(req);
2839}
2840
Chris Wilson8d9fc7f2014-02-25 17:11:23 +02002841struct drm_i915_gem_request *
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002842i915_gem_find_active_request(struct intel_engine_cs *engine)
Chris Wilson9375e442010-09-19 12:21:28 +01002843{
Chris Wilson4db080f2013-12-04 11:37:09 +00002844 struct drm_i915_gem_request *request;
Mika Kuoppalaaa60c662013-06-12 15:13:20 +03002845
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002846 list_for_each_entry(request, &engine->request_list, list) {
John Harrison1b5a4332014-11-24 18:49:42 +00002847 if (i915_gem_request_completed(request, false))
Chris Wilson4db080f2013-12-04 11:37:09 +00002848 continue;
Mika Kuoppalaaa60c662013-06-12 15:13:20 +03002849
Mika Kuoppalab6b0fac2014-01-30 19:04:43 +02002850 return request;
Chris Wilson4db080f2013-12-04 11:37:09 +00002851 }
Mika Kuoppalab6b0fac2014-01-30 19:04:43 +02002852
2853 return NULL;
2854}
2855
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00002856static void i915_gem_reset_engine_status(struct drm_i915_private *dev_priv,
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002857 struct intel_engine_cs *engine)
Mika Kuoppalab6b0fac2014-01-30 19:04:43 +02002858{
2859 struct drm_i915_gem_request *request;
2860 bool ring_hung;
2861
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002862 request = i915_gem_find_active_request(engine);
Mika Kuoppalab6b0fac2014-01-30 19:04:43 +02002863
2864 if (request == NULL)
2865 return;
2866
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002867 ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
Mika Kuoppalab6b0fac2014-01-30 19:04:43 +02002868
Mika Kuoppala939fd762014-01-30 19:04:44 +02002869 i915_set_reset_status(dev_priv, request->ctx, ring_hung);
Mika Kuoppalab6b0fac2014-01-30 19:04:43 +02002870
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002871 list_for_each_entry_continue(request, &engine->request_list, list)
Mika Kuoppala939fd762014-01-30 19:04:44 +02002872 i915_set_reset_status(dev_priv, request->ctx, false);
Chris Wilson4db080f2013-12-04 11:37:09 +00002873}
2874
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00002875static void i915_gem_reset_engine_cleanup(struct drm_i915_private *dev_priv,
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002876 struct intel_engine_cs *engine)
Chris Wilson4db080f2013-12-04 11:37:09 +00002877{
Chris Wilson608c1a52015-09-03 13:01:40 +01002878 struct intel_ringbuffer *buffer;
2879
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002880 while (!list_empty(&engine->active_list)) {
Chris Wilson05394f32010-11-08 19:18:58 +00002881 struct drm_i915_gem_object *obj;
Eric Anholt673a3942008-07-30 12:06:12 -07002882
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002883 obj = list_first_entry(&engine->active_list,
Chris Wilson05394f32010-11-08 19:18:58 +00002884 struct drm_i915_gem_object,
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00002885 engine_list[engine->id]);
Eric Anholt673a3942008-07-30 12:06:12 -07002886
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002887 i915_gem_object_retire__read(obj, engine->id);
Eric Anholt673a3942008-07-30 12:06:12 -07002888 }
Ben Widawsky1d62bee2014-01-01 10:15:13 -08002889
2890 /*
Oscar Mateodcb4c122014-11-13 10:28:10 +00002891 * Clear the execlists queue up before freeing the requests, as those
2892 * are the ones that keep the context and ringbuffer backing objects
2893 * pinned in place.
2894 */
Oscar Mateodcb4c122014-11-13 10:28:10 +00002895
Tomas Elf7de16912015-10-19 16:32:32 +01002896 if (i915.enable_execlists) {
Tvrtko Ursulin27af5ee2016-04-04 12:11:56 +01002897 /* Ensure irq handler finishes or is cancelled. */
2898 tasklet_kill(&engine->irq_tasklet);
Mika Kuoppala1197b4f2015-01-13 11:32:24 +02002899
Tvrtko Ursulin27af5ee2016-04-04 12:11:56 +01002900 spin_lock_bh(&engine->execlist_lock);
Tomas Elfc5baa562015-10-23 18:02:37 +01002901 /* list_splice_tail_init checks for empty lists */
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002902 list_splice_tail_init(&engine->execlist_queue,
2903 &engine->execlist_retired_req_list);
Tvrtko Ursulin27af5ee2016-04-04 12:11:56 +01002904 spin_unlock_bh(&engine->execlist_lock);
Mika Kuoppala1197b4f2015-01-13 11:32:24 +02002905
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002906 intel_execlists_retire_requests(engine);
Oscar Mateodcb4c122014-11-13 10:28:10 +00002907 }
2908
2909 /*
Ben Widawsky1d62bee2014-01-01 10:15:13 -08002910 * We must free the requests after all the corresponding objects have
2911 * been moved off active lists. Which is the same order as the normal
2912 * retire_requests function does. This is important if object hold
2913 * implicit references on things like e.g. ppgtt address spaces through
2914 * the request.
2915 */
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002916 while (!list_empty(&engine->request_list)) {
Ben Widawsky1d62bee2014-01-01 10:15:13 -08002917 struct drm_i915_gem_request *request;
2918
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002919 request = list_first_entry(&engine->request_list,
Ben Widawsky1d62bee2014-01-01 10:15:13 -08002920 struct drm_i915_gem_request,
2921 list);
2922
Chris Wilsonb4716182015-04-27 13:41:17 +01002923 i915_gem_request_retire(request);
Ben Widawsky1d62bee2014-01-01 10:15:13 -08002924 }
Chris Wilson608c1a52015-09-03 13:01:40 +01002925
2926 /* Having flushed all requests from all queues, we know that all
2927 * ringbuffers must now be empty. However, since we do not reclaim
2928 * all space when retiring the request (to prevent HEADs colliding
2929 * with rapid ringbuffer wraparound) the amount of available space
2930 * upon reset is less than when we start. Do one more pass over
2931 * all the ringbuffers to reset last_retired_head.
2932 */
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002933 list_for_each_entry(buffer, &engine->buffers, link) {
Chris Wilson608c1a52015-09-03 13:01:40 +01002934 buffer->last_retired_head = buffer->tail;
2935 intel_ring_update_space(buffer);
2936 }
Chris Wilson2ed53a92016-04-07 07:29:11 +01002937
2938 intel_ring_init_seqno(engine, engine->last_submitted_seqno);
Eric Anholt673a3942008-07-30 12:06:12 -07002939}
2940
Chris Wilson069efc12010-09-30 16:53:18 +01002941void i915_gem_reset(struct drm_device *dev)
Eric Anholt673a3942008-07-30 12:06:12 -07002942{
Chris Wilsondfaae392010-09-22 10:31:52 +01002943 struct drm_i915_private *dev_priv = dev->dev_private;
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00002944 struct intel_engine_cs *engine;
Eric Anholt673a3942008-07-30 12:06:12 -07002945
Chris Wilson4db080f2013-12-04 11:37:09 +00002946 /*
2947 * Before we free the objects from the requests, we need to inspect
2948 * them for finding the guilty party. As the requests only borrow
2949 * their reference to the objects, the inspection must be done first.
2950 */
Dave Gordonb4ac5af2016-03-24 11:20:38 +00002951 for_each_engine(engine, dev_priv)
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00002952 i915_gem_reset_engine_status(dev_priv, engine);
Chris Wilson4db080f2013-12-04 11:37:09 +00002953
Dave Gordonb4ac5af2016-03-24 11:20:38 +00002954 for_each_engine(engine, dev_priv)
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00002955 i915_gem_reset_engine_cleanup(dev_priv, engine);
Chris Wilsondfaae392010-09-22 10:31:52 +01002956
Ben Widawskyacce9ff2013-12-06 14:11:03 -08002957 i915_gem_context_reset(dev);
2958
Chris Wilson19b2dbd2013-06-12 10:15:12 +01002959 i915_gem_restore_fences(dev);
Chris Wilsonb4716182015-04-27 13:41:17 +01002960
2961 WARN_ON(i915_verify_lists(dev));
Eric Anholt673a3942008-07-30 12:06:12 -07002962}
2963
2964/**
2965 * This function clears the request list as sequence numbers are passed.
2966 */
Chris Wilson1cf0ba12014-05-05 09:07:33 +01002967void
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002968i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
Eric Anholt673a3942008-07-30 12:06:12 -07002969{
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002970 WARN_ON(i915_verify_lists(engine->dev));
Eric Anholt673a3942008-07-30 12:06:12 -07002971
Chris Wilson832a3aa2015-03-18 18:19:22 +00002972 /* Retire requests first as we use it above for the early return.
2973 * If we retire requests last, we may use a later seqno and so clear
2974 * the requests lists without clearing the active list, leading to
2975 * confusion.
Chris Wilsone9103032014-01-07 11:45:14 +00002976 */
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002977 while (!list_empty(&engine->request_list)) {
Eric Anholt673a3942008-07-30 12:06:12 -07002978 struct drm_i915_gem_request *request;
Eric Anholt673a3942008-07-30 12:06:12 -07002979
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002980 request = list_first_entry(&engine->request_list,
Eric Anholt673a3942008-07-30 12:06:12 -07002981 struct drm_i915_gem_request,
2982 list);
Eric Anholt673a3942008-07-30 12:06:12 -07002983
John Harrison1b5a4332014-11-24 18:49:42 +00002984 if (!i915_gem_request_completed(request, true))
Eric Anholt673a3942008-07-30 12:06:12 -07002985 break;
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002986
Chris Wilsonb4716182015-04-27 13:41:17 +01002987 i915_gem_request_retire(request);
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002988 }
2989
Chris Wilson832a3aa2015-03-18 18:19:22 +00002990 /* Move any buffers on the active list that are no longer referenced
2991 * by the ringbuffer to the flushing/inactive lists as appropriate,
2992 * before we free the context associated with the requests.
2993 */
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002994 while (!list_empty(&engine->active_list)) {
Chris Wilson832a3aa2015-03-18 18:19:22 +00002995 struct drm_i915_gem_object *obj;
2996
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002997 obj = list_first_entry(&engine->active_list,
2998 struct drm_i915_gem_object,
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00002999 engine_list[engine->id]);
Chris Wilson832a3aa2015-03-18 18:19:22 +00003000
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00003001 if (!list_empty(&obj->last_read_req[engine->id]->list))
Chris Wilson832a3aa2015-03-18 18:19:22 +00003002 break;
3003
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00003004 i915_gem_object_retire__read(obj, engine->id);
Chris Wilson832a3aa2015-03-18 18:19:22 +00003005 }
3006
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00003007 if (unlikely(engine->trace_irq_req &&
3008 i915_gem_request_completed(engine->trace_irq_req, true))) {
3009 engine->irq_put(engine);
3010 i915_gem_request_assign(&engine->trace_irq_req, NULL);
Chris Wilson9d34e5d2009-09-24 05:26:06 +01003011 }
Chris Wilson23bc5982010-09-29 16:10:57 +01003012
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00003013 WARN_ON(i915_verify_lists(engine->dev));
Eric Anholt673a3942008-07-30 12:06:12 -07003014}
3015
Chris Wilsonb29c19b2013-09-25 17:34:56 +01003016bool
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01003017i915_gem_retire_requests(struct drm_device *dev)
3018{
Jani Nikula3e31c6c2014-03-31 14:27:16 +03003019 struct drm_i915_private *dev_priv = dev->dev_private;
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00003020 struct intel_engine_cs *engine;
Chris Wilsonb29c19b2013-09-25 17:34:56 +01003021 bool idle = true;
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01003022
Dave Gordonb4ac5af2016-03-24 11:20:38 +00003023 for_each_engine(engine, dev_priv) {
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00003024 i915_gem_retire_requests_ring(engine);
3025 idle &= list_empty(&engine->request_list);
Thomas Danielc86ee3a92014-11-13 10:27:05 +00003026 if (i915.enable_execlists) {
Tvrtko Ursulin27af5ee2016-04-04 12:11:56 +01003027 spin_lock_bh(&engine->execlist_lock);
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00003028 idle &= list_empty(&engine->execlist_queue);
Tvrtko Ursulin27af5ee2016-04-04 12:11:56 +01003029 spin_unlock_bh(&engine->execlist_lock);
Thomas Danielc86ee3a92014-11-13 10:27:05 +00003030
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00003031 intel_execlists_retire_requests(engine);
Thomas Danielc86ee3a92014-11-13 10:27:05 +00003032 }
Chris Wilsonb29c19b2013-09-25 17:34:56 +01003033 }
3034
3035 if (idle)
3036 mod_delayed_work(dev_priv->wq,
3037 &dev_priv->mm.idle_work,
3038 msecs_to_jiffies(100));
3039
3040 return idle;
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01003041}
3042
Daniel Vetter75ef9da2010-08-21 00:25:16 +02003043static void
Eric Anholt673a3942008-07-30 12:06:12 -07003044i915_gem_retire_work_handler(struct work_struct *work)
3045{
Chris Wilsonb29c19b2013-09-25 17:34:56 +01003046 struct drm_i915_private *dev_priv =
3047 container_of(work, typeof(*dev_priv), mm.retire_work.work);
3048 struct drm_device *dev = dev_priv->dev;
Chris Wilson0a587052011-01-09 21:05:44 +00003049 bool idle;
Eric Anholt673a3942008-07-30 12:06:12 -07003050
Chris Wilson891b48c2010-09-29 12:26:37 +01003051 /* Come back later if the device is busy... */
Chris Wilsonb29c19b2013-09-25 17:34:56 +01003052 idle = false;
3053 if (mutex_trylock(&dev->struct_mutex)) {
3054 idle = i915_gem_retire_requests(dev);
3055 mutex_unlock(&dev->struct_mutex);
3056 }
3057 if (!idle)
Chris Wilsonbcb45082012-10-05 17:02:57 +01003058 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
3059 round_jiffies_up_relative(HZ));
Chris Wilsonb29c19b2013-09-25 17:34:56 +01003060}
Chris Wilson891b48c2010-09-29 12:26:37 +01003061
Chris Wilsonb29c19b2013-09-25 17:34:56 +01003062static void
3063i915_gem_idle_work_handler(struct work_struct *work)
3064{
3065 struct drm_i915_private *dev_priv =
3066 container_of(work, typeof(*dev_priv), mm.idle_work.work);
Chris Wilson35c94182015-04-07 16:20:37 +01003067 struct drm_device *dev = dev_priv->dev;
Dave Gordonb4ac5af2016-03-24 11:20:38 +00003068 struct intel_engine_cs *engine;
Zou Nan haid1b851f2010-05-21 09:08:57 +08003069
Dave Gordonb4ac5af2016-03-24 11:20:38 +00003070 for_each_engine(engine, dev_priv)
3071 if (!list_empty(&engine->request_list))
Chris Wilson423795c2015-04-07 16:21:08 +01003072 return;
Zou Nan hai852835f2010-05-21 09:08:56 +08003073
Daniel Vetter30ecad72015-12-09 09:29:36 +01003074 /* we probably should sync with hangcheck here, using cancel_work_sync.
Dave Gordonb4ac5af2016-03-24 11:20:38 +00003075 * Also locking seems to be fubar here, engine->request_list is protected
Daniel Vetter30ecad72015-12-09 09:29:36 +01003076 * by dev->struct_mutex. */
3077
Chris Wilson35c94182015-04-07 16:20:37 +01003078 intel_mark_idle(dev);
3079
3080 if (mutex_trylock(&dev->struct_mutex)) {
Dave Gordonb4ac5af2016-03-24 11:20:38 +00003081 for_each_engine(engine, dev_priv)
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00003082 i915_gem_batch_pool_fini(&engine->batch_pool);
Chris Wilson35c94182015-04-07 16:20:37 +01003083
3084 mutex_unlock(&dev->struct_mutex);
3085 }
Eric Anholt673a3942008-07-30 12:06:12 -07003086}
3087
Ben Widawsky5816d642012-04-11 11:18:19 -07003088/**
Daniel Vetter30dfebf2012-06-01 15:21:23 +02003089 * Ensures that an object will eventually get non-busy by flushing any required
3090 * write domains, emitting any outstanding lazy request and retiring and
3091 * completed requests.
3092 */
3093static int
3094i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
3095{
John Harrisona5ac0f92015-05-29 17:44:15 +01003096 int i;
Daniel Vetter30dfebf2012-06-01 15:21:23 +02003097
Chris Wilsonb4716182015-04-27 13:41:17 +01003098 if (!obj->active)
3099 return 0;
John Harrison41c52412014-11-24 18:49:43 +00003100
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00003101 for (i = 0; i < I915_NUM_ENGINES; i++) {
Chris Wilsonb4716182015-04-27 13:41:17 +01003102 struct drm_i915_gem_request *req;
3103
3104 req = obj->last_read_req[i];
3105 if (req == NULL)
3106 continue;
3107
3108 if (list_empty(&req->list))
3109 goto retire;
3110
Chris Wilsonb4716182015-04-27 13:41:17 +01003111 if (i915_gem_request_completed(req, true)) {
3112 __i915_gem_request_retire__upto(req);
3113retire:
3114 i915_gem_object_retire__read(obj, i);
3115 }
Daniel Vetter30dfebf2012-06-01 15:21:23 +02003116 }
3117
3118 return 0;
3119}
3120
3121/**
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003122 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
3123 * @DRM_IOCTL_ARGS: standard ioctl arguments
3124 *
3125 * Returns 0 if successful, else an error is returned with the remaining time in
3126 * the timeout parameter.
3127 * -ETIME: object is still busy after timeout
3128 * -ERESTARTSYS: signal interrupted the wait
3129 * -ENONENT: object doesn't exist
3130 * Also possible, but rare:
3131 * -EAGAIN: GPU wedged
3132 * -ENOMEM: damn
3133 * -ENODEV: Internal IRQ fail
3134 * -E?: The add request failed
3135 *
3136 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
3137 * non-zero timeout parameter the wait ioctl will wait for the given number of
3138 * nanoseconds on an object becoming unbusy. Since the wait itself does so
3139 * without holding struct_mutex the object may become re-busied before this
3140 * function completes. A similar but shorter * race condition exists in the busy
3141 * ioctl
3142 */
3143int
3144i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3145{
Jani Nikula3e31c6c2014-03-31 14:27:16 +03003146 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003147 struct drm_i915_gem_wait *args = data;
3148 struct drm_i915_gem_object *obj;
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00003149 struct drm_i915_gem_request *req[I915_NUM_ENGINES];
Daniel Vetterf69061b2012-12-06 09:01:42 +01003150 unsigned reset_counter;
Chris Wilsonb4716182015-04-27 13:41:17 +01003151 int i, n = 0;
3152 int ret;
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003153
Daniel Vetter11b5d512014-09-29 15:31:26 +02003154 if (args->flags != 0)
3155 return -EINVAL;
3156
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003157 ret = i915_mutex_lock_interruptible(dev);
3158 if (ret)
3159 return ret;
3160
3161 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
3162 if (&obj->base == NULL) {
3163 mutex_unlock(&dev->struct_mutex);
3164 return -ENOENT;
3165 }
3166
Daniel Vetter30dfebf2012-06-01 15:21:23 +02003167 /* Need to make sure the object gets inactive eventually. */
3168 ret = i915_gem_object_flush_active(obj);
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003169 if (ret)
3170 goto out;
3171
Chris Wilsonb4716182015-04-27 13:41:17 +01003172 if (!obj->active)
John Harrison97b2a6a2014-11-24 18:49:26 +00003173 goto out;
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003174
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003175 /* Do this after OLR check to make sure we make forward progress polling
Chris Wilson762e4582015-03-04 18:09:26 +00003176 * on this IOCTL with a timeout == 0 (like busy ioctl)
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003177 */
Chris Wilson762e4582015-03-04 18:09:26 +00003178 if (args->timeout_ns == 0) {
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003179 ret = -ETIME;
3180 goto out;
3181 }
3182
3183 drm_gem_object_unreference(&obj->base);
Daniel Vetterf69061b2012-12-06 09:01:42 +01003184 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
Chris Wilsonb4716182015-04-27 13:41:17 +01003185
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00003186 for (i = 0; i < I915_NUM_ENGINES; i++) {
Chris Wilsonb4716182015-04-27 13:41:17 +01003187 if (obj->last_read_req[i] == NULL)
3188 continue;
3189
3190 req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
3191 }
3192
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003193 mutex_unlock(&dev->struct_mutex);
3194
Chris Wilsonb4716182015-04-27 13:41:17 +01003195 for (i = 0; i < n; i++) {
3196 if (ret == 0)
3197 ret = __i915_wait_request(req[i], reset_counter, true,
3198 args->timeout_ns > 0 ? &args->timeout_ns : NULL,
Chris Wilsonb6aa0872015-12-02 09:13:46 +00003199 to_rps_client(file));
Chris Wilsonb4716182015-04-27 13:41:17 +01003200 i915_gem_request_unreference__unlocked(req[i]);
3201 }
John Harrisonff865882014-11-24 18:49:28 +00003202 return ret;
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003203
3204out:
3205 drm_gem_object_unreference(&obj->base);
3206 mutex_unlock(&dev->struct_mutex);
3207 return ret;
3208}
3209
Chris Wilsonb4716182015-04-27 13:41:17 +01003210static int
3211__i915_gem_object_sync(struct drm_i915_gem_object *obj,
3212 struct intel_engine_cs *to,
John Harrison91af1272015-06-18 13:14:56 +01003213 struct drm_i915_gem_request *from_req,
3214 struct drm_i915_gem_request **to_req)
Chris Wilsonb4716182015-04-27 13:41:17 +01003215{
3216 struct intel_engine_cs *from;
3217 int ret;
3218
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00003219 from = i915_gem_request_get_engine(from_req);
Chris Wilsonb4716182015-04-27 13:41:17 +01003220 if (to == from)
3221 return 0;
3222
John Harrison91af1272015-06-18 13:14:56 +01003223 if (i915_gem_request_completed(from_req, true))
Chris Wilsonb4716182015-04-27 13:41:17 +01003224 return 0;
3225
Chris Wilsonb4716182015-04-27 13:41:17 +01003226 if (!i915_semaphore_is_enabled(obj->base.dev)) {
Chris Wilsona6f766f2015-04-27 13:41:20 +01003227 struct drm_i915_private *i915 = to_i915(obj->base.dev);
John Harrison91af1272015-06-18 13:14:56 +01003228 ret = __i915_wait_request(from_req,
Chris Wilsona6f766f2015-04-27 13:41:20 +01003229 atomic_read(&i915->gpu_error.reset_counter),
3230 i915->mm.interruptible,
3231 NULL,
3232 &i915->rps.semaphores);
Chris Wilsonb4716182015-04-27 13:41:17 +01003233 if (ret)
3234 return ret;
3235
John Harrison91af1272015-06-18 13:14:56 +01003236 i915_gem_object_retire_request(obj, from_req);
Chris Wilsonb4716182015-04-27 13:41:17 +01003237 } else {
3238 int idx = intel_ring_sync_index(from, to);
John Harrison91af1272015-06-18 13:14:56 +01003239 u32 seqno = i915_gem_request_get_seqno(from_req);
3240
3241 WARN_ON(!to_req);
Chris Wilsonb4716182015-04-27 13:41:17 +01003242
3243 if (seqno <= from->semaphore.sync_seqno[idx])
3244 return 0;
3245
John Harrison91af1272015-06-18 13:14:56 +01003246 if (*to_req == NULL) {
Dave Gordon26827082016-01-19 19:02:53 +00003247 struct drm_i915_gem_request *req;
3248
3249 req = i915_gem_request_alloc(to, NULL);
3250 if (IS_ERR(req))
3251 return PTR_ERR(req);
3252
3253 *to_req = req;
John Harrison91af1272015-06-18 13:14:56 +01003254 }
3255
John Harrison599d9242015-05-29 17:44:04 +01003256 trace_i915_gem_ring_sync_to(*to_req, from, from_req);
3257 ret = to->semaphore.sync_to(*to_req, from, seqno);
Chris Wilsonb4716182015-04-27 13:41:17 +01003258 if (ret)
3259 return ret;
3260
3261 /* We use last_read_req because sync_to()
3262 * might have just caused seqno wrap under
3263 * the radar.
3264 */
3265 from->semaphore.sync_seqno[idx] =
3266 i915_gem_request_get_seqno(obj->last_read_req[from->id]);
3267 }
3268
3269 return 0;
3270}
3271
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003272/**
Ben Widawsky5816d642012-04-11 11:18:19 -07003273 * i915_gem_object_sync - sync an object to a ring.
3274 *
3275 * @obj: object which may be in use on another ring.
3276 * @to: ring we wish to use the object on. May be NULL.
John Harrison91af1272015-06-18 13:14:56 +01003277 * @to_req: request we wish to use the object for. See below.
3278 * This will be allocated and returned if a request is
3279 * required but not passed in.
Ben Widawsky5816d642012-04-11 11:18:19 -07003280 *
3281 * This code is meant to abstract object synchronization with the GPU.
3282 * Calling with NULL implies synchronizing the object with the CPU
Chris Wilsonb4716182015-04-27 13:41:17 +01003283 * rather than a particular GPU ring. Conceptually we serialise writes
John Harrison91af1272015-06-18 13:14:56 +01003284 * between engines inside the GPU. We only allow one engine to write
Chris Wilsonb4716182015-04-27 13:41:17 +01003285 * into a buffer at any time, but multiple readers. To ensure each has
3286 * a coherent view of memory, we must:
3287 *
3288 * - If there is an outstanding write request to the object, the new
3289 * request must wait for it to complete (either CPU or in hw, requests
3290 * on the same ring will be naturally ordered).
3291 *
3292 * - If we are a write request (pending_write_domain is set), the new
3293 * request must wait for outstanding read requests to complete.
Ben Widawsky5816d642012-04-11 11:18:19 -07003294 *
John Harrison91af1272015-06-18 13:14:56 +01003295 * For CPU synchronisation (NULL to) no request is required. For syncing with
3296 * rings to_req must be non-NULL. However, a request does not have to be
3297 * pre-allocated. If *to_req is NULL and sync commands will be emitted then a
3298 * request will be allocated automatically and returned through *to_req. Note
3299 * that it is not guaranteed that commands will be emitted (because the system
3300 * might already be idle). Hence there is no need to create a request that
3301 * might never have any work submitted. Note further that if a request is
3302 * returned in *to_req, it is the responsibility of the caller to submit
3303 * that request (after potentially adding more work to it).
3304 *
Ben Widawsky5816d642012-04-11 11:18:19 -07003305 * Returns 0 if successful, else propagates up the lower layer error.
3306 */
Ben Widawsky2911a352012-04-05 14:47:36 -07003307int
3308i915_gem_object_sync(struct drm_i915_gem_object *obj,
John Harrison91af1272015-06-18 13:14:56 +01003309 struct intel_engine_cs *to,
3310 struct drm_i915_gem_request **to_req)
Ben Widawsky2911a352012-04-05 14:47:36 -07003311{
Chris Wilsonb4716182015-04-27 13:41:17 +01003312 const bool readonly = obj->base.pending_write_domain == 0;
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00003313 struct drm_i915_gem_request *req[I915_NUM_ENGINES];
Chris Wilsonb4716182015-04-27 13:41:17 +01003314 int ret, i, n;
Ben Widawsky2911a352012-04-05 14:47:36 -07003315
Chris Wilsonb4716182015-04-27 13:41:17 +01003316 if (!obj->active)
Ben Widawsky2911a352012-04-05 14:47:36 -07003317 return 0;
3318
Chris Wilsonb4716182015-04-27 13:41:17 +01003319 if (to == NULL)
3320 return i915_gem_object_wait_rendering(obj, readonly);
Ben Widawsky2911a352012-04-05 14:47:36 -07003321
Chris Wilsonb4716182015-04-27 13:41:17 +01003322 n = 0;
3323 if (readonly) {
3324 if (obj->last_write_req)
3325 req[n++] = obj->last_write_req;
3326 } else {
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00003327 for (i = 0; i < I915_NUM_ENGINES; i++)
Chris Wilsonb4716182015-04-27 13:41:17 +01003328 if (obj->last_read_req[i])
3329 req[n++] = obj->last_read_req[i];
3330 }
3331 for (i = 0; i < n; i++) {
John Harrison91af1272015-06-18 13:14:56 +01003332 ret = __i915_gem_object_sync(obj, to, req[i], to_req);
Chris Wilsonb4716182015-04-27 13:41:17 +01003333 if (ret)
3334 return ret;
3335 }
Ben Widawsky2911a352012-04-05 14:47:36 -07003336
Chris Wilsonb4716182015-04-27 13:41:17 +01003337 return 0;
Ben Widawsky2911a352012-04-05 14:47:36 -07003338}
3339
Chris Wilsonb5ffc9b2011-04-13 22:06:03 +01003340static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
3341{
3342 u32 old_write_domain, old_read_domains;
3343
Chris Wilsonb5ffc9b2011-04-13 22:06:03 +01003344 /* Force a pagefault for domain tracking on next user access */
3345 i915_gem_release_mmap(obj);
3346
Keith Packardb97c3d92011-06-24 21:02:59 -07003347 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3348 return;
3349
Chris Wilson97c809fd2012-10-09 19:24:38 +01003350 /* Wait for any direct GTT access to complete */
3351 mb();
3352
Chris Wilsonb5ffc9b2011-04-13 22:06:03 +01003353 old_read_domains = obj->base.read_domains;
3354 old_write_domain = obj->base.write_domain;
3355
3356 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
3357 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
3358
3359 trace_i915_gem_object_change_domain(obj,
3360 old_read_domains,
3361 old_write_domain);
3362}
3363
Tvrtko Ursuline9f24d52015-10-05 13:26:36 +01003364static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
Eric Anholt673a3942008-07-30 12:06:12 -07003365{
Ben Widawsky07fe0b12013-07-31 17:00:10 -07003366 struct drm_i915_gem_object *obj = vma->obj;
Jani Nikula3e31c6c2014-03-31 14:27:16 +03003367 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
Chris Wilson43e28f02013-01-08 10:53:09 +00003368 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003369
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00003370 if (list_empty(&vma->obj_link))
Eric Anholt673a3942008-07-30 12:06:12 -07003371 return 0;
3372
Daniel Vetter0ff501c2013-08-29 19:50:31 +02003373 if (!drm_mm_node_allocated(&vma->node)) {
3374 i915_gem_vma_destroy(vma);
Daniel Vetter0ff501c2013-08-29 19:50:31 +02003375 return 0;
3376 }
Ben Widawsky433544b2013-08-13 18:09:06 -07003377
Ben Widawskyd7f46fc2013-12-06 14:10:55 -08003378 if (vma->pin_count)
Chris Wilson31d8d652012-05-24 19:11:20 +01003379 return -EBUSY;
Eric Anholt673a3942008-07-30 12:06:12 -07003380
Chris Wilsonc4670ad2012-08-20 10:23:27 +01003381 BUG_ON(obj->pages == NULL);
3382
Tvrtko Ursuline9f24d52015-10-05 13:26:36 +01003383 if (wait) {
3384 ret = i915_gem_object_wait_rendering(obj, false);
3385 if (ret)
3386 return ret;
3387 }
Chris Wilsona8198ee2011-04-13 22:04:09 +01003388
Chris Wilson596c5922016-02-26 11:03:20 +00003389 if (vma->is_ggtt && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
Daniel Vetter8b1bc9b2014-02-14 14:06:07 +01003390 i915_gem_object_finish_gtt(obj);
Chris Wilsona8198ee2011-04-13 22:04:09 +01003391
Daniel Vetter8b1bc9b2014-02-14 14:06:07 +01003392 /* release the fence reg _after_ flushing */
3393 ret = i915_gem_object_put_fence(obj);
3394 if (ret)
3395 return ret;
3396 }
Daniel Vetter96b47b62009-12-15 17:50:00 +01003397
Ben Widawsky07fe0b12013-07-31 17:00:10 -07003398 trace_i915_vma_unbind(vma);
Chris Wilsondb53a302011-02-03 11:57:46 +00003399
Daniel Vetter777dc5b2015-04-14 17:35:12 +02003400 vma->vm->unbind_vma(vma);
Mika Kuoppala5e562f12015-04-30 11:02:31 +03003401 vma->bound = 0;
Ben Widawsky6f65e292013-12-06 14:10:56 -08003402
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00003403 list_del_init(&vma->vm_link);
Chris Wilson596c5922016-02-26 11:03:20 +00003404 if (vma->is_ggtt) {
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00003405 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3406 obj->map_and_fenceable = false;
3407 } else if (vma->ggtt_view.pages) {
3408 sg_free_table(vma->ggtt_view.pages);
3409 kfree(vma->ggtt_view.pages);
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00003410 }
Chris Wilson016a65a2015-06-11 08:06:08 +01003411 vma->ggtt_view.pages = NULL;
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00003412 }
Eric Anholt673a3942008-07-30 12:06:12 -07003413
Ben Widawsky2f633152013-07-17 12:19:03 -07003414 drm_mm_remove_node(&vma->node);
3415 i915_gem_vma_destroy(vma);
3416
3417 /* Since the unbound list is global, only move to that list if
Daniel Vetterb93dab62013-08-26 11:23:47 +02003418 * no more VMAs exist. */
Imre Deake2273302015-07-09 12:59:05 +03003419 if (list_empty(&obj->vma_list))
Ben Widawsky2f633152013-07-17 12:19:03 -07003420 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
Eric Anholt673a3942008-07-30 12:06:12 -07003421
Chris Wilson70903c32013-12-04 09:59:09 +00003422 /* And finally now the object is completely decoupled from this vma,
3423 * we can drop its hold on the backing storage and allow it to be
3424 * reaped by the shrinker.
3425 */
3426 i915_gem_object_unpin_pages(obj);
3427
Chris Wilson88241782011-01-07 17:09:48 +00003428 return 0;
Chris Wilson54cf91d2010-11-25 18:00:26 +00003429}
3430
Tvrtko Ursuline9f24d52015-10-05 13:26:36 +01003431int i915_vma_unbind(struct i915_vma *vma)
3432{
3433 return __i915_vma_unbind(vma, true);
3434}
3435
3436int __i915_vma_unbind_no_wait(struct i915_vma *vma)
3437{
3438 return __i915_vma_unbind(vma, false);
3439}
3440
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07003441int i915_gpu_idle(struct drm_device *dev)
Daniel Vetter4df2faf2010-02-19 11:52:00 +01003442{
Jani Nikula3e31c6c2014-03-31 14:27:16 +03003443 struct drm_i915_private *dev_priv = dev->dev_private;
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00003444 struct intel_engine_cs *engine;
Dave Gordonb4ac5af2016-03-24 11:20:38 +00003445 int ret;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01003446
Daniel Vetter4df2faf2010-02-19 11:52:00 +01003447 /* Flush everything onto the inactive list. */
Dave Gordonb4ac5af2016-03-24 11:20:38 +00003448 for_each_engine(engine, dev_priv) {
Thomas Danielecdb5fd2014-08-20 16:29:24 +01003449 if (!i915.enable_execlists) {
John Harrison73cfa862015-05-29 17:43:35 +01003450 struct drm_i915_gem_request *req;
3451
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00003452 req = i915_gem_request_alloc(engine, NULL);
Dave Gordon26827082016-01-19 19:02:53 +00003453 if (IS_ERR(req))
3454 return PTR_ERR(req);
John Harrison73cfa862015-05-29 17:43:35 +01003455
John Harrisonba01cc92015-05-29 17:43:41 +01003456 ret = i915_switch_context(req);
John Harrison73cfa862015-05-29 17:43:35 +01003457 if (ret) {
3458 i915_gem_request_cancel(req);
3459 return ret;
3460 }
3461
John Harrison75289872015-05-29 17:43:49 +01003462 i915_add_request_no_flush(req);
Thomas Danielecdb5fd2014-08-20 16:29:24 +01003463 }
Ben Widawskyb6c74882012-08-14 14:35:14 -07003464
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00003465 ret = intel_engine_idle(engine);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00003466 if (ret)
3467 return ret;
3468 }
Zou Nan haid1b851f2010-05-21 09:08:57 +08003469
Chris Wilsonb4716182015-04-27 13:41:17 +01003470 WARN_ON(i915_verify_lists(dev));
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01003471 return 0;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01003472}
3473
Chris Wilson4144f9b2014-09-11 08:43:48 +01003474static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
Chris Wilson42d6ab42012-07-26 11:49:32 +01003475 unsigned long cache_level)
3476{
Chris Wilson4144f9b2014-09-11 08:43:48 +01003477 struct drm_mm_node *gtt_space = &vma->node;
Chris Wilson42d6ab42012-07-26 11:49:32 +01003478 struct drm_mm_node *other;
3479
Chris Wilson4144f9b2014-09-11 08:43:48 +01003480 /*
3481 * On some machines we have to be careful when putting differing types
3482 * of snoopable memory together to avoid the prefetcher crossing memory
3483 * domains and dying. During vm initialisation, we decide whether or not
3484 * these constraints apply and set the drm_mm.color_adjust
3485 * appropriately.
Chris Wilson42d6ab42012-07-26 11:49:32 +01003486 */
Chris Wilson4144f9b2014-09-11 08:43:48 +01003487 if (vma->vm->mm.color_adjust == NULL)
Chris Wilson42d6ab42012-07-26 11:49:32 +01003488 return true;
3489
Ben Widawskyc6cfb322013-07-05 14:41:06 -07003490 if (!drm_mm_node_allocated(gtt_space))
Chris Wilson42d6ab42012-07-26 11:49:32 +01003491 return true;
3492
3493 if (list_empty(&gtt_space->node_list))
3494 return true;
3495
3496 other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3497 if (other->allocated && !other->hole_follows && other->color != cache_level)
3498 return false;
3499
3500 other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3501 if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3502 return false;
3503
3504 return true;
3505}
3506
Jesse Barnesde151cf2008-11-12 10:03:55 -08003507/**
Joonas Lahtinen91e67112015-05-06 14:33:58 +03003508 * Finds free space in the GTT aperture and binds the object or a view of it
3509 * there.
Eric Anholt673a3942008-07-30 12:06:12 -07003510 */
Daniel Vetter262de142014-02-14 14:01:20 +01003511static struct i915_vma *
Ben Widawsky07fe0b12013-07-31 17:00:10 -07003512i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3513 struct i915_address_space *vm,
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02003514 const struct i915_ggtt_view *ggtt_view,
Ben Widawsky07fe0b12013-07-31 17:00:10 -07003515 unsigned alignment,
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02003516 uint64_t flags)
Eric Anholt673a3942008-07-30 12:06:12 -07003517{
Chris Wilson05394f32010-11-08 19:18:58 +00003518 struct drm_device *dev = obj->base.dev;
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03003519 struct drm_i915_private *dev_priv = to_i915(dev);
3520 struct i915_ggtt *ggtt = &dev_priv->ggtt;
Michel Thierry65bd3422015-07-29 17:23:58 +01003521 u32 fence_alignment, unfenced_alignment;
Michel Thierry101b5062015-10-01 13:33:57 +01003522 u32 search_flag, alloc_flag;
3523 u64 start, end;
Michel Thierry65bd3422015-07-29 17:23:58 +01003524 u64 size, fence_size;
Ben Widawsky2f633152013-07-17 12:19:03 -07003525 struct i915_vma *vma;
Chris Wilson07f73f62009-09-14 16:50:30 +01003526 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003527
Joonas Lahtinen91e67112015-05-06 14:33:58 +03003528 if (i915_is_ggtt(vm)) {
3529 u32 view_size;
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02003530
Joonas Lahtinen91e67112015-05-06 14:33:58 +03003531 if (WARN_ON(!ggtt_view))
3532 return ERR_PTR(-EINVAL);
3533
3534 view_size = i915_ggtt_view_size(obj, ggtt_view);
3535
3536 fence_size = i915_gem_get_gtt_size(dev,
3537 view_size,
3538 obj->tiling_mode);
3539 fence_alignment = i915_gem_get_gtt_alignment(dev,
3540 view_size,
3541 obj->tiling_mode,
3542 true);
3543 unfenced_alignment = i915_gem_get_gtt_alignment(dev,
3544 view_size,
3545 obj->tiling_mode,
3546 false);
3547 size = flags & PIN_MAPPABLE ? fence_size : view_size;
3548 } else {
3549 fence_size = i915_gem_get_gtt_size(dev,
3550 obj->base.size,
3551 obj->tiling_mode);
3552 fence_alignment = i915_gem_get_gtt_alignment(dev,
3553 obj->base.size,
3554 obj->tiling_mode,
3555 true);
3556 unfenced_alignment =
3557 i915_gem_get_gtt_alignment(dev,
3558 obj->base.size,
3559 obj->tiling_mode,
3560 false);
3561 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
3562 }
Chris Wilsona00b10c2010-09-24 21:15:47 +01003563
Michel Thierry101b5062015-10-01 13:33:57 +01003564 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3565 end = vm->total;
3566 if (flags & PIN_MAPPABLE)
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03003567 end = min_t(u64, end, ggtt->mappable_end);
Michel Thierry101b5062015-10-01 13:33:57 +01003568 if (flags & PIN_ZONE_4G)
Michel Thierry48ea1e32016-01-11 11:39:27 +00003569 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
Michel Thierry101b5062015-10-01 13:33:57 +01003570
Eric Anholt673a3942008-07-30 12:06:12 -07003571 if (alignment == 0)
Daniel Vetter1ec9e262014-02-14 14:01:11 +01003572 alignment = flags & PIN_MAPPABLE ? fence_alignment :
Daniel Vetter5e783302010-11-14 22:32:36 +01003573 unfenced_alignment;
Daniel Vetter1ec9e262014-02-14 14:01:11 +01003574 if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
Joonas Lahtinen91e67112015-05-06 14:33:58 +03003575 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
3576 ggtt_view ? ggtt_view->type : 0,
3577 alignment);
Daniel Vetter262de142014-02-14 14:01:20 +01003578 return ERR_PTR(-EINVAL);
Eric Anholt673a3942008-07-30 12:06:12 -07003579 }
3580
Joonas Lahtinen91e67112015-05-06 14:33:58 +03003581 /* If binding the object/GGTT view requires more space than the entire
3582 * aperture has, reject it early before evicting everything in a vain
3583 * attempt to find space.
Chris Wilson654fc602010-05-27 13:18:21 +01003584 */
Joonas Lahtinen91e67112015-05-06 14:33:58 +03003585 if (size > end) {
Michel Thierry65bd3422015-07-29 17:23:58 +01003586 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
Joonas Lahtinen91e67112015-05-06 14:33:58 +03003587 ggtt_view ? ggtt_view->type : 0,
3588 size,
Daniel Vetter1ec9e262014-02-14 14:01:11 +01003589 flags & PIN_MAPPABLE ? "mappable" : "total",
Chris Wilsond23db882014-05-23 08:48:08 +02003590 end);
Daniel Vetter262de142014-02-14 14:01:20 +01003591 return ERR_PTR(-E2BIG);
Chris Wilson654fc602010-05-27 13:18:21 +01003592 }
3593
Chris Wilson37e680a2012-06-07 15:38:42 +01003594 ret = i915_gem_object_get_pages(obj);
Chris Wilson6c085a72012-08-20 11:40:46 +02003595 if (ret)
Daniel Vetter262de142014-02-14 14:01:20 +01003596 return ERR_PTR(ret);
Chris Wilson6c085a72012-08-20 11:40:46 +02003597
Chris Wilsonfbdda6f2012-11-20 10:45:16 +00003598 i915_gem_object_pin_pages(obj);
3599
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02003600 vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
3601 i915_gem_obj_lookup_or_create_vma(obj, vm);
3602
Daniel Vetter262de142014-02-14 14:01:20 +01003603 if (IS_ERR(vma))
Daniel Vetterbc6bc152013-07-22 12:12:38 +02003604 goto err_unpin;
Ben Widawsky2f633152013-07-17 12:19:03 -07003605
Chris Wilson506a8e82015-12-08 11:55:07 +00003606 if (flags & PIN_OFFSET_FIXED) {
3607 uint64_t offset = flags & PIN_OFFSET_MASK;
3608
3609 if (offset & (alignment - 1) || offset + size > end) {
3610 ret = -EINVAL;
3611 goto err_free_vma;
3612 }
3613 vma->node.start = offset;
3614 vma->node.size = size;
3615 vma->node.color = obj->cache_level;
3616 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3617 if (ret) {
3618 ret = i915_gem_evict_for_vma(vma);
3619 if (ret == 0)
3620 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3621 }
3622 if (ret)
3623 goto err_free_vma;
Michel Thierry101b5062015-10-01 13:33:57 +01003624 } else {
Chris Wilson506a8e82015-12-08 11:55:07 +00003625 if (flags & PIN_HIGH) {
3626 search_flag = DRM_MM_SEARCH_BELOW;
3627 alloc_flag = DRM_MM_CREATE_TOP;
3628 } else {
3629 search_flag = DRM_MM_SEARCH_DEFAULT;
3630 alloc_flag = DRM_MM_CREATE_DEFAULT;
3631 }
Michel Thierry101b5062015-10-01 13:33:57 +01003632
Ben Widawsky0a9ae0d2013-05-25 12:26:35 -07003633search_free:
Chris Wilson506a8e82015-12-08 11:55:07 +00003634 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3635 size, alignment,
3636 obj->cache_level,
3637 start, end,
3638 search_flag,
3639 alloc_flag);
3640 if (ret) {
3641 ret = i915_gem_evict_something(dev, vm, size, alignment,
3642 obj->cache_level,
3643 start, end,
3644 flags);
3645 if (ret == 0)
3646 goto search_free;
Chris Wilson97311292009-09-21 00:22:34 +01003647
Chris Wilson506a8e82015-12-08 11:55:07 +00003648 goto err_free_vma;
3649 }
Chris Wilsondc9dd7a2012-12-07 20:37:07 +00003650 }
Chris Wilson4144f9b2014-09-11 08:43:48 +01003651 if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
Ben Widawsky2f633152013-07-17 12:19:03 -07003652 ret = -EINVAL;
Daniel Vetterbc6bc152013-07-22 12:12:38 +02003653 goto err_remove_node;
Eric Anholt673a3942008-07-30 12:06:12 -07003654 }
3655
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00003656 trace_i915_vma_bind(vma, flags);
Daniel Vetter08755462015-04-20 09:04:05 -07003657 ret = i915_vma_bind(vma, obj->cache_level, flags);
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00003658 if (ret)
Imre Deake2273302015-07-09 12:59:05 +03003659 goto err_remove_node;
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00003660
Ben Widawsky35c20a62013-05-31 11:28:48 -07003661 list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00003662 list_add_tail(&vma->vm_link, &vm->inactive_list);
Chris Wilsonbf1a1092010-08-07 11:01:20 +01003663
Daniel Vetter262de142014-02-14 14:01:20 +01003664 return vma;
Ben Widawsky2f633152013-07-17 12:19:03 -07003665
Daniel Vetterbc6bc152013-07-22 12:12:38 +02003666err_remove_node:
Dan Carpenter6286ef92013-07-19 08:46:27 +03003667 drm_mm_remove_node(&vma->node);
Daniel Vetterbc6bc152013-07-22 12:12:38 +02003668err_free_vma:
Ben Widawsky2f633152013-07-17 12:19:03 -07003669 i915_gem_vma_destroy(vma);
Daniel Vetter262de142014-02-14 14:01:20 +01003670 vma = ERR_PTR(ret);
Daniel Vetterbc6bc152013-07-22 12:12:38 +02003671err_unpin:
Ben Widawsky2f633152013-07-17 12:19:03 -07003672 i915_gem_object_unpin_pages(obj);
Daniel Vetter262de142014-02-14 14:01:20 +01003673 return vma;
Eric Anholt673a3942008-07-30 12:06:12 -07003674}
3675
Chris Wilson000433b2013-08-08 14:41:09 +01003676bool
Chris Wilson2c225692013-08-09 12:26:45 +01003677i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3678 bool force)
Eric Anholt673a3942008-07-30 12:06:12 -07003679{
Eric Anholt673a3942008-07-30 12:06:12 -07003680 /* If we don't have a page list set up, then we're not pinned
3681 * to GPU, and we can ignore the cache flush because it'll happen
3682 * again at bind time.
3683 */
Chris Wilson05394f32010-11-08 19:18:58 +00003684 if (obj->pages == NULL)
Chris Wilson000433b2013-08-08 14:41:09 +01003685 return false;
Eric Anholt673a3942008-07-30 12:06:12 -07003686
Imre Deak769ce462013-02-13 21:56:05 +02003687 /*
3688 * Stolen memory is always coherent with the GPU as it is explicitly
3689 * marked as wc by the system, or the system is cache-coherent.
3690 */
Chris Wilson6a2c4232014-11-04 04:51:40 -08003691 if (obj->stolen || obj->phys_handle)
Chris Wilson000433b2013-08-08 14:41:09 +01003692 return false;
Imre Deak769ce462013-02-13 21:56:05 +02003693
Chris Wilson9c23f7f2011-03-29 16:59:52 -07003694 /* If the GPU is snooping the contents of the CPU cache,
3695 * we do not need to manually clear the CPU cache lines. However,
3696 * the caches are only snooped when the render cache is
3697 * flushed/invalidated. As we always have to emit invalidations
3698 * and flushes when moving into and out of the RENDER domain, correct
3699 * snooping behaviour occurs naturally as the result of our domain
3700 * tracking.
3701 */
Chris Wilson0f719792015-01-13 13:32:52 +00003702 if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3703 obj->cache_dirty = true;
Chris Wilson000433b2013-08-08 14:41:09 +01003704 return false;
Chris Wilson0f719792015-01-13 13:32:52 +00003705 }
Chris Wilson9c23f7f2011-03-29 16:59:52 -07003706
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003707 trace_i915_gem_object_clflush(obj);
Chris Wilson9da3da62012-06-01 15:20:22 +01003708 drm_clflush_sg(obj->pages);
Chris Wilson0f719792015-01-13 13:32:52 +00003709 obj->cache_dirty = false;
Chris Wilson000433b2013-08-08 14:41:09 +01003710
3711 return true;
Eric Anholte47c68e2008-11-14 13:35:19 -08003712}
3713
3714/** Flushes the GTT write domain for the object if it's dirty. */
3715static void
Chris Wilson05394f32010-11-08 19:18:58 +00003716i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
Eric Anholte47c68e2008-11-14 13:35:19 -08003717{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003718 uint32_t old_write_domain;
3719
Chris Wilson05394f32010-11-08 19:18:58 +00003720 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
Eric Anholte47c68e2008-11-14 13:35:19 -08003721 return;
3722
Chris Wilson63256ec2011-01-04 18:42:07 +00003723 /* No actual flushing is required for the GTT write domain. Writes
Eric Anholte47c68e2008-11-14 13:35:19 -08003724 * to it immediately go to main memory as far as we know, so there's
3725 * no chipset flush. It also doesn't land in render cache.
Chris Wilson63256ec2011-01-04 18:42:07 +00003726 *
3727 * However, we do have to enforce the order so that all writes through
3728 * the GTT land before any writes to the device, such as updates to
3729 * the GATT itself.
Eric Anholte47c68e2008-11-14 13:35:19 -08003730 */
Chris Wilson63256ec2011-01-04 18:42:07 +00003731 wmb();
3732
Chris Wilson05394f32010-11-08 19:18:58 +00003733 old_write_domain = obj->base.write_domain;
3734 obj->base.write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003735
Rodrigo Vivide152b62015-07-07 16:28:51 -07003736 intel_fb_obj_flush(obj, false, ORIGIN_GTT);
Daniel Vetterf99d7062014-06-19 16:01:59 +02003737
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003738 trace_i915_gem_object_change_domain(obj,
Chris Wilson05394f32010-11-08 19:18:58 +00003739 obj->base.read_domains,
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003740 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08003741}
3742
3743/** Flushes the CPU write domain for the object if it's dirty. */
3744static void
Daniel Vettere62b59e2015-01-21 14:53:48 +01003745i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
Eric Anholte47c68e2008-11-14 13:35:19 -08003746{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003747 uint32_t old_write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08003748
Chris Wilson05394f32010-11-08 19:18:58 +00003749 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
Eric Anholte47c68e2008-11-14 13:35:19 -08003750 return;
3751
Daniel Vettere62b59e2015-01-21 14:53:48 +01003752 if (i915_gem_clflush_object(obj, obj->pin_display))
Chris Wilson000433b2013-08-08 14:41:09 +01003753 i915_gem_chipset_flush(obj->base.dev);
3754
Chris Wilson05394f32010-11-08 19:18:58 +00003755 old_write_domain = obj->base.write_domain;
3756 obj->base.write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003757
Rodrigo Vivide152b62015-07-07 16:28:51 -07003758 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
Daniel Vetterf99d7062014-06-19 16:01:59 +02003759
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003760 trace_i915_gem_object_change_domain(obj,
Chris Wilson05394f32010-11-08 19:18:58 +00003761 obj->base.read_domains,
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003762 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08003763}
3764
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003765/**
3766 * Moves a single object to the GTT read, and possibly write domain.
3767 *
3768 * This function returns when the move is complete, including waiting on
3769 * flushes to occur.
3770 */
Jesse Barnes79e53942008-11-07 14:24:08 -08003771int
Chris Wilson20217462010-11-23 15:26:33 +00003772i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003773{
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03003774 struct drm_device *dev = obj->base.dev;
3775 struct drm_i915_private *dev_priv = to_i915(dev);
3776 struct i915_ggtt *ggtt = &dev_priv->ggtt;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003777 uint32_t old_write_domain, old_read_domains;
Chris Wilson43566de2015-01-02 16:29:29 +05303778 struct i915_vma *vma;
Eric Anholte47c68e2008-11-14 13:35:19 -08003779 int ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003780
Chris Wilson8d7e3de2011-02-07 15:23:02 +00003781 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3782 return 0;
3783
Chris Wilson0201f1e2012-07-20 12:41:01 +01003784 ret = i915_gem_object_wait_rendering(obj, !write);
Chris Wilson88241782011-01-07 17:09:48 +00003785 if (ret)
3786 return ret;
3787
Chris Wilson43566de2015-01-02 16:29:29 +05303788 /* Flush and acquire obj->pages so that we are coherent through
3789 * direct access in memory with previous cached writes through
3790 * shmemfs and that our cache domain tracking remains valid.
3791 * For example, if the obj->filp was moved to swap without us
3792 * being notified and releasing the pages, we would mistakenly
3793 * continue to assume that the obj remained out of the CPU cached
3794 * domain.
3795 */
3796 ret = i915_gem_object_get_pages(obj);
3797 if (ret)
3798 return ret;
3799
Daniel Vettere62b59e2015-01-21 14:53:48 +01003800 i915_gem_object_flush_cpu_write_domain(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003801
Chris Wilsond0a57782012-10-09 19:24:37 +01003802 /* Serialise direct access to this object with the barriers for
3803 * coherent writes from the GPU, by effectively invalidating the
3804 * GTT domain upon first access.
3805 */
3806 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3807 mb();
3808
Chris Wilson05394f32010-11-08 19:18:58 +00003809 old_write_domain = obj->base.write_domain;
3810 old_read_domains = obj->base.read_domains;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003811
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003812 /* It should now be out of any other write domains, and we can update
3813 * the domain values for our changes.
3814 */
Chris Wilson05394f32010-11-08 19:18:58 +00003815 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3816 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08003817 if (write) {
Chris Wilson05394f32010-11-08 19:18:58 +00003818 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3819 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3820 obj->dirty = 1;
Eric Anholte47c68e2008-11-14 13:35:19 -08003821 }
3822
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003823 trace_i915_gem_object_change_domain(obj,
3824 old_read_domains,
3825 old_write_domain);
3826
Chris Wilson8325a092012-04-24 15:52:35 +01003827 /* And bump the LRU for this access */
Chris Wilson43566de2015-01-02 16:29:29 +05303828 vma = i915_gem_obj_to_ggtt(obj);
3829 if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00003830 list_move_tail(&vma->vm_link,
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03003831 &ggtt->base.inactive_list);
Chris Wilson8325a092012-04-24 15:52:35 +01003832
Eric Anholte47c68e2008-11-14 13:35:19 -08003833 return 0;
3834}
3835
Chris Wilsonef55f922015-10-09 14:11:27 +01003836/**
3837 * Changes the cache-level of an object across all VMA.
3838 *
3839 * After this function returns, the object will be in the new cache-level
3840 * across all GTT and the contents of the backing storage will be coherent,
3841 * with respect to the new cache-level. In order to keep the backing storage
3842 * coherent for all users, we only allow a single cache level to be set
3843 * globally on the object and prevent it from being changed whilst the
3844 * hardware is reading from the object. That is if the object is currently
3845 * on the scanout it will be set to uncached (or equivalent display
3846 * cache coherency) and all non-MOCS GPU access will also be uncached so
3847 * that all direct access to the scanout remains coherent.
3848 */
Chris Wilsone4ffd172011-04-04 09:44:39 +01003849int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3850 enum i915_cache_level cache_level)
3851{
Daniel Vetter7bddb012012-02-09 17:15:47 +01003852 struct drm_device *dev = obj->base.dev;
Chris Wilsondf6f7832014-03-21 07:40:56 +00003853 struct i915_vma *vma, *next;
Chris Wilsonef55f922015-10-09 14:11:27 +01003854 bool bound = false;
Ville Syrjäläed75a552015-08-11 19:47:10 +03003855 int ret = 0;
Chris Wilsone4ffd172011-04-04 09:44:39 +01003856
3857 if (obj->cache_level == cache_level)
Ville Syrjäläed75a552015-08-11 19:47:10 +03003858 goto out;
Chris Wilsone4ffd172011-04-04 09:44:39 +01003859
Chris Wilsonef55f922015-10-09 14:11:27 +01003860 /* Inspect the list of currently bound VMA and unbind any that would
3861 * be invalid given the new cache-level. This is principally to
3862 * catch the issue of the CS prefetch crossing page boundaries and
3863 * reading an invalid PTE on older architectures.
3864 */
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00003865 list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
Chris Wilsonef55f922015-10-09 14:11:27 +01003866 if (!drm_mm_node_allocated(&vma->node))
3867 continue;
3868
3869 if (vma->pin_count) {
3870 DRM_DEBUG("can not change the cache level of pinned objects\n");
3871 return -EBUSY;
3872 }
3873
Chris Wilson4144f9b2014-09-11 08:43:48 +01003874 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
Ben Widawsky07fe0b12013-07-31 17:00:10 -07003875 ret = i915_vma_unbind(vma);
Ben Widawsky3089c6f2013-07-31 17:00:03 -07003876 if (ret)
3877 return ret;
Chris Wilsonef55f922015-10-09 14:11:27 +01003878 } else
3879 bound = true;
Chris Wilson42d6ab42012-07-26 11:49:32 +01003880 }
3881
Chris Wilsonef55f922015-10-09 14:11:27 +01003882 /* We can reuse the existing drm_mm nodes but need to change the
3883 * cache-level on the PTE. We could simply unbind them all and
3884 * rebind with the correct cache-level on next use. However since
3885 * we already have a valid slot, dma mapping, pages etc, we may as
3886 * rewrite the PTE in the belief that doing so tramples upon less
3887 * state and so involves less work.
3888 */
3889 if (bound) {
3890 /* Before we change the PTE, the GPU must not be accessing it.
3891 * If we wait upon the object, we know that all the bound
3892 * VMA are no longer active.
3893 */
Chris Wilson2e2f3512015-04-27 13:41:14 +01003894 ret = i915_gem_object_wait_rendering(obj, false);
Chris Wilsone4ffd172011-04-04 09:44:39 +01003895 if (ret)
3896 return ret;
3897
Chris Wilsonef55f922015-10-09 14:11:27 +01003898 if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
3899 /* Access to snoopable pages through the GTT is
3900 * incoherent and on some machines causes a hard
3901 * lockup. Relinquish the CPU mmaping to force
3902 * userspace to refault in the pages and we can
3903 * then double check if the GTT mapping is still
3904 * valid for that pointer access.
3905 */
3906 i915_gem_release_mmap(obj);
Chris Wilsone4ffd172011-04-04 09:44:39 +01003907
Chris Wilsonef55f922015-10-09 14:11:27 +01003908 /* As we no longer need a fence for GTT access,
3909 * we can relinquish it now (and so prevent having
3910 * to steal a fence from someone else on the next
3911 * fence request). Note GPU activity would have
3912 * dropped the fence as all snoopable access is
3913 * supposed to be linear.
3914 */
Chris Wilsone4ffd172011-04-04 09:44:39 +01003915 ret = i915_gem_object_put_fence(obj);
3916 if (ret)
3917 return ret;
Chris Wilsonef55f922015-10-09 14:11:27 +01003918 } else {
3919 /* We either have incoherent backing store and
3920 * so no GTT access or the architecture is fully
3921 * coherent. In such cases, existing GTT mmaps
3922 * ignore the cache bit in the PTE and we can
3923 * rewrite it without confusing the GPU or having
3924 * to force userspace to fault back in its mmaps.
3925 */
Chris Wilsone4ffd172011-04-04 09:44:39 +01003926 }
3927
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00003928 list_for_each_entry(vma, &obj->vma_list, obj_link) {
Chris Wilsonef55f922015-10-09 14:11:27 +01003929 if (!drm_mm_node_allocated(&vma->node))
3930 continue;
3931
3932 ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
3933 if (ret)
3934 return ret;
3935 }
Chris Wilsone4ffd172011-04-04 09:44:39 +01003936 }
3937
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00003938 list_for_each_entry(vma, &obj->vma_list, obj_link)
Chris Wilson2c225692013-08-09 12:26:45 +01003939 vma->node.color = cache_level;
3940 obj->cache_level = cache_level;
3941
Ville Syrjäläed75a552015-08-11 19:47:10 +03003942out:
Chris Wilsonef55f922015-10-09 14:11:27 +01003943 /* Flush the dirty CPU caches to the backing storage so that the
3944 * object is now coherent at its new cache level (with respect
3945 * to the access domain).
3946 */
Chris Wilson0f719792015-01-13 13:32:52 +00003947 if (obj->cache_dirty &&
3948 obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
3949 cpu_write_needs_clflush(obj)) {
3950 if (i915_gem_clflush_object(obj, true))
3951 i915_gem_chipset_flush(obj->base.dev);
Chris Wilsone4ffd172011-04-04 09:44:39 +01003952 }
3953
Chris Wilsone4ffd172011-04-04 09:44:39 +01003954 return 0;
3955}
3956
Ben Widawsky199adf42012-09-21 17:01:20 -07003957int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3958 struct drm_file *file)
Chris Wilsone6994ae2012-07-10 10:27:08 +01003959{
Ben Widawsky199adf42012-09-21 17:01:20 -07003960 struct drm_i915_gem_caching *args = data;
Chris Wilsone6994ae2012-07-10 10:27:08 +01003961 struct drm_i915_gem_object *obj;
Chris Wilsone6994ae2012-07-10 10:27:08 +01003962
3963 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilson432be692015-05-07 12:14:55 +01003964 if (&obj->base == NULL)
3965 return -ENOENT;
Chris Wilsone6994ae2012-07-10 10:27:08 +01003966
Chris Wilson651d7942013-08-08 14:41:10 +01003967 switch (obj->cache_level) {
3968 case I915_CACHE_LLC:
3969 case I915_CACHE_L3_LLC:
3970 args->caching = I915_CACHING_CACHED;
3971 break;
3972
Chris Wilson4257d3b2013-08-08 14:41:11 +01003973 case I915_CACHE_WT:
3974 args->caching = I915_CACHING_DISPLAY;
3975 break;
3976
Chris Wilson651d7942013-08-08 14:41:10 +01003977 default:
3978 args->caching = I915_CACHING_NONE;
3979 break;
3980 }
Chris Wilsone6994ae2012-07-10 10:27:08 +01003981
Chris Wilson432be692015-05-07 12:14:55 +01003982 drm_gem_object_unreference_unlocked(&obj->base);
3983 return 0;
Chris Wilsone6994ae2012-07-10 10:27:08 +01003984}
3985
Ben Widawsky199adf42012-09-21 17:01:20 -07003986int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3987 struct drm_file *file)
Chris Wilsone6994ae2012-07-10 10:27:08 +01003988{
Imre Deakfd0fe6a2015-11-04 21:25:32 +02003989 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawsky199adf42012-09-21 17:01:20 -07003990 struct drm_i915_gem_caching *args = data;
Chris Wilsone6994ae2012-07-10 10:27:08 +01003991 struct drm_i915_gem_object *obj;
3992 enum i915_cache_level level;
3993 int ret;
3994
Ben Widawsky199adf42012-09-21 17:01:20 -07003995 switch (args->caching) {
3996 case I915_CACHING_NONE:
Chris Wilsone6994ae2012-07-10 10:27:08 +01003997 level = I915_CACHE_NONE;
3998 break;
Ben Widawsky199adf42012-09-21 17:01:20 -07003999 case I915_CACHING_CACHED:
Imre Deake5756c12015-08-14 18:43:30 +03004000 /*
4001 * Due to a HW issue on BXT A stepping, GPU stores via a
4002 * snooped mapping may leave stale data in a corresponding CPU
4003 * cacheline, whereas normally such cachelines would get
4004 * invalidated.
4005 */
Tvrtko Ursulinca377802016-03-02 12:10:31 +00004006 if (!HAS_LLC(dev) && !HAS_SNOOP(dev))
Imre Deake5756c12015-08-14 18:43:30 +03004007 return -ENODEV;
4008
Chris Wilsone6994ae2012-07-10 10:27:08 +01004009 level = I915_CACHE_LLC;
4010 break;
Chris Wilson4257d3b2013-08-08 14:41:11 +01004011 case I915_CACHING_DISPLAY:
4012 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
4013 break;
Chris Wilsone6994ae2012-07-10 10:27:08 +01004014 default:
4015 return -EINVAL;
4016 }
4017
Imre Deakfd0fe6a2015-11-04 21:25:32 +02004018 intel_runtime_pm_get(dev_priv);
4019
Ben Widawsky3bc29132012-09-26 16:15:20 -07004020 ret = i915_mutex_lock_interruptible(dev);
4021 if (ret)
Imre Deakfd0fe6a2015-11-04 21:25:32 +02004022 goto rpm_put;
Ben Widawsky3bc29132012-09-26 16:15:20 -07004023
Chris Wilsone6994ae2012-07-10 10:27:08 +01004024 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4025 if (&obj->base == NULL) {
4026 ret = -ENOENT;
4027 goto unlock;
4028 }
4029
4030 ret = i915_gem_object_set_cache_level(obj, level);
4031
4032 drm_gem_object_unreference(&obj->base);
4033unlock:
4034 mutex_unlock(&dev->struct_mutex);
Imre Deakfd0fe6a2015-11-04 21:25:32 +02004035rpm_put:
4036 intel_runtime_pm_put(dev_priv);
4037
Chris Wilsone6994ae2012-07-10 10:27:08 +01004038 return ret;
4039}
4040
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004041/*
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004042 * Prepare buffer for display plane (scanout, cursors, etc).
4043 * Can be called from an uninterruptible phase (modesetting) and allows
4044 * any flushes to be pipelined (for pageflips).
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004045 */
4046int
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004047i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
4048 u32 alignment,
Tvrtko Ursuline6617332015-03-23 11:10:33 +00004049 const struct i915_ggtt_view *view)
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004050{
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004051 u32 old_read_domains, old_write_domain;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004052 int ret;
4053
Chris Wilsoncc98b412013-08-09 12:25:09 +01004054 /* Mark the pin_display early so that we account for the
4055 * display coherency whilst setting up the cache domains.
4056 */
Tvrtko Ursulin8a0c39b2015-04-13 11:50:09 +01004057 obj->pin_display++;
Chris Wilsoncc98b412013-08-09 12:25:09 +01004058
Eric Anholta7ef0642011-03-29 16:59:54 -07004059 /* The display engine is not coherent with the LLC cache on gen6. As
4060 * a result, we make sure that the pinning that is about to occur is
4061 * done with uncached PTEs. This is lowest common denominator for all
4062 * chipsets.
4063 *
4064 * However for gen6+, we could do better by using the GFDT bit instead
4065 * of uncaching, which would allow us to flush all the LLC-cached data
4066 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
4067 */
Chris Wilson651d7942013-08-08 14:41:10 +01004068 ret = i915_gem_object_set_cache_level(obj,
4069 HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
Eric Anholta7ef0642011-03-29 16:59:54 -07004070 if (ret)
Chris Wilsoncc98b412013-08-09 12:25:09 +01004071 goto err_unpin_display;
Eric Anholta7ef0642011-03-29 16:59:54 -07004072
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004073 /* As the user may map the buffer once pinned in the display plane
4074 * (e.g. libkms for the bootup splash), we have to ensure that we
4075 * always use map_and_fenceable for all scanout buffers.
4076 */
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00004077 ret = i915_gem_object_ggtt_pin(obj, view, alignment,
4078 view->type == I915_GGTT_VIEW_NORMAL ?
4079 PIN_MAPPABLE : 0);
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004080 if (ret)
Chris Wilsoncc98b412013-08-09 12:25:09 +01004081 goto err_unpin_display;
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004082
Daniel Vettere62b59e2015-01-21 14:53:48 +01004083 i915_gem_object_flush_cpu_write_domain(obj);
Chris Wilsonb118c1e2010-05-27 13:18:14 +01004084
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004085 old_write_domain = obj->base.write_domain;
Chris Wilson05394f32010-11-08 19:18:58 +00004086 old_read_domains = obj->base.read_domains;
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004087
4088 /* It should now be out of any other write domains, and we can update
4089 * the domain values for our changes.
4090 */
Chris Wilsone5f1d962012-07-20 12:41:00 +01004091 obj->base.write_domain = 0;
Chris Wilson05394f32010-11-08 19:18:58 +00004092 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004093
4094 trace_i915_gem_object_change_domain(obj,
4095 old_read_domains,
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004096 old_write_domain);
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004097
4098 return 0;
Chris Wilsoncc98b412013-08-09 12:25:09 +01004099
4100err_unpin_display:
Tvrtko Ursulin8a0c39b2015-04-13 11:50:09 +01004101 obj->pin_display--;
Chris Wilsoncc98b412013-08-09 12:25:09 +01004102 return ret;
4103}
4104
4105void
Tvrtko Ursuline6617332015-03-23 11:10:33 +00004106i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
4107 const struct i915_ggtt_view *view)
Chris Wilsoncc98b412013-08-09 12:25:09 +01004108{
Tvrtko Ursulin8a0c39b2015-04-13 11:50:09 +01004109 if (WARN_ON(obj->pin_display == 0))
4110 return;
4111
Tvrtko Ursuline6617332015-03-23 11:10:33 +00004112 i915_gem_object_ggtt_unpin_view(obj, view);
4113
Tvrtko Ursulin8a0c39b2015-04-13 11:50:09 +01004114 obj->pin_display--;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004115}
4116
Eric Anholte47c68e2008-11-14 13:35:19 -08004117/**
4118 * Moves a single object to the CPU read, and possibly write domain.
4119 *
4120 * This function returns when the move is complete, including waiting on
4121 * flushes to occur.
4122 */
Chris Wilsondabdfe02012-03-26 10:10:27 +02004123int
Chris Wilson919926a2010-11-12 13:42:53 +00004124i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
Eric Anholte47c68e2008-11-14 13:35:19 -08004125{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01004126 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08004127 int ret;
4128
Chris Wilson8d7e3de2011-02-07 15:23:02 +00004129 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
4130 return 0;
4131
Chris Wilson0201f1e2012-07-20 12:41:01 +01004132 ret = i915_gem_object_wait_rendering(obj, !write);
Chris Wilson88241782011-01-07 17:09:48 +00004133 if (ret)
4134 return ret;
4135
Eric Anholte47c68e2008-11-14 13:35:19 -08004136 i915_gem_object_flush_gtt_write_domain(obj);
4137
Chris Wilson05394f32010-11-08 19:18:58 +00004138 old_write_domain = obj->base.write_domain;
4139 old_read_domains = obj->base.read_domains;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01004140
Eric Anholte47c68e2008-11-14 13:35:19 -08004141 /* Flush the CPU cache if it's still invalid. */
Chris Wilson05394f32010-11-08 19:18:58 +00004142 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
Chris Wilson2c225692013-08-09 12:26:45 +01004143 i915_gem_clflush_object(obj, false);
Eric Anholte47c68e2008-11-14 13:35:19 -08004144
Chris Wilson05394f32010-11-08 19:18:58 +00004145 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
Eric Anholte47c68e2008-11-14 13:35:19 -08004146 }
4147
4148 /* It should now be out of any other write domains, and we can update
4149 * the domain values for our changes.
4150 */
Chris Wilson05394f32010-11-08 19:18:58 +00004151 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
Eric Anholte47c68e2008-11-14 13:35:19 -08004152
4153 /* If we're writing through the CPU, then the GPU read domains will
4154 * need to be invalidated at next use.
4155 */
4156 if (write) {
Chris Wilson05394f32010-11-08 19:18:58 +00004157 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4158 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
Eric Anholte47c68e2008-11-14 13:35:19 -08004159 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -08004160
Chris Wilson1c5d22f2009-08-25 11:15:50 +01004161 trace_i915_gem_object_change_domain(obj,
4162 old_read_domains,
4163 old_write_domain);
4164
Eric Anholt2ef7eea2008-11-10 10:53:25 -08004165 return 0;
4166}
4167
Eric Anholt673a3942008-07-30 12:06:12 -07004168/* Throttle our rendering by waiting until the ring has completed our requests
4169 * emitted over 20 msec ago.
4170 *
Eric Anholtb9624422009-06-03 07:27:35 +00004171 * Note that if we were to use the current jiffies each time around the loop,
4172 * we wouldn't escape the function with any frames outstanding if the time to
4173 * render a frame was over 20ms.
4174 *
Eric Anholt673a3942008-07-30 12:06:12 -07004175 * This should get us reasonable parallelism between CPU and GPU but also
4176 * relatively low latency when blocking on a particular request to finish.
4177 */
4178static int
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004179i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07004180{
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004181 struct drm_i915_private *dev_priv = dev->dev_private;
4182 struct drm_i915_file_private *file_priv = file->driver_priv;
Chris Wilsond0bc54f2015-05-21 21:01:48 +01004183 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
John Harrison54fb2412014-11-24 18:49:27 +00004184 struct drm_i915_gem_request *request, *target = NULL;
Daniel Vetterf69061b2012-12-06 09:01:42 +01004185 unsigned reset_counter;
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004186 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004187
Daniel Vetter308887a2012-11-14 17:14:06 +01004188 ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
4189 if (ret)
4190 return ret;
4191
4192 ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
4193 if (ret)
4194 return ret;
Chris Wilsone110e8d2011-01-26 15:39:14 +00004195
Chris Wilson1c255952010-09-26 11:03:27 +01004196 spin_lock(&file_priv->mm.lock);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004197 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
Eric Anholtb9624422009-06-03 07:27:35 +00004198 if (time_after_eq(request->emitted_jiffies, recent_enough))
4199 break;
4200
John Harrisonfcfa423c2015-05-29 17:44:12 +01004201 /*
4202 * Note that the request might not have been submitted yet.
4203 * In which case emitted_jiffies will be zero.
4204 */
4205 if (!request->emitted_jiffies)
4206 continue;
4207
John Harrison54fb2412014-11-24 18:49:27 +00004208 target = request;
Eric Anholtb9624422009-06-03 07:27:35 +00004209 }
Daniel Vetterf69061b2012-12-06 09:01:42 +01004210 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
John Harrisonff865882014-11-24 18:49:28 +00004211 if (target)
4212 i915_gem_request_reference(target);
Chris Wilson1c255952010-09-26 11:03:27 +01004213 spin_unlock(&file_priv->mm.lock);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004214
John Harrison54fb2412014-11-24 18:49:27 +00004215 if (target == NULL)
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004216 return 0;
4217
John Harrison9c654812014-11-24 18:49:35 +00004218 ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004219 if (ret == 0)
4220 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
Eric Anholtb9624422009-06-03 07:27:35 +00004221
Chris Wilson41037f92015-03-27 11:01:36 +00004222 i915_gem_request_unreference__unlocked(target);
John Harrisonff865882014-11-24 18:49:28 +00004223
Eric Anholt673a3942008-07-30 12:06:12 -07004224 return ret;
4225}
4226
Chris Wilsond23db882014-05-23 08:48:08 +02004227static bool
4228i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
4229{
4230 struct drm_i915_gem_object *obj = vma->obj;
4231
4232 if (alignment &&
4233 vma->node.start & (alignment - 1))
4234 return true;
4235
4236 if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
4237 return true;
4238
4239 if (flags & PIN_OFFSET_BIAS &&
4240 vma->node.start < (flags & PIN_OFFSET_MASK))
4241 return true;
4242
Chris Wilson506a8e82015-12-08 11:55:07 +00004243 if (flags & PIN_OFFSET_FIXED &&
4244 vma->node.start != (flags & PIN_OFFSET_MASK))
4245 return true;
4246
Chris Wilsond23db882014-05-23 08:48:08 +02004247 return false;
4248}
4249
Chris Wilsond0710ab2015-11-20 14:16:39 +00004250void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
4251{
4252 struct drm_i915_gem_object *obj = vma->obj;
4253 bool mappable, fenceable;
4254 u32 fence_size, fence_alignment;
4255
4256 fence_size = i915_gem_get_gtt_size(obj->base.dev,
4257 obj->base.size,
4258 obj->tiling_mode);
4259 fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
4260 obj->base.size,
4261 obj->tiling_mode,
4262 true);
4263
4264 fenceable = (vma->node.size == fence_size &&
4265 (vma->node.start & (fence_alignment - 1)) == 0);
4266
4267 mappable = (vma->node.start + fence_size <=
Joonas Lahtinen62106b42016-03-18 10:42:57 +02004268 to_i915(obj->base.dev)->ggtt.mappable_end);
Chris Wilsond0710ab2015-11-20 14:16:39 +00004269
4270 obj->map_and_fenceable = mappable && fenceable;
4271}
4272
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004273static int
4274i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
4275 struct i915_address_space *vm,
4276 const struct i915_ggtt_view *ggtt_view,
4277 uint32_t alignment,
4278 uint64_t flags)
Eric Anholt673a3942008-07-30 12:06:12 -07004279{
Ben Widawsky6e7186a2014-05-06 22:21:36 -07004280 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
Ben Widawsky07fe0b12013-07-31 17:00:10 -07004281 struct i915_vma *vma;
Chris Wilsonef79e172014-10-31 13:53:52 +00004282 unsigned bound;
Eric Anholt673a3942008-07-30 12:06:12 -07004283 int ret;
4284
Ben Widawsky6e7186a2014-05-06 22:21:36 -07004285 if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
4286 return -ENODEV;
4287
Daniel Vetterbf3d1492014-02-14 14:01:12 +01004288 if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
Daniel Vetter1ec9e262014-02-14 14:01:11 +01004289 return -EINVAL;
Ben Widawsky07fe0b12013-07-31 17:00:10 -07004290
Chris Wilsonc826c442014-10-31 13:53:53 +00004291 if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
4292 return -EINVAL;
4293
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004294 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
4295 return -EINVAL;
4296
4297 vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
4298 i915_gem_obj_to_vma(obj, vm);
4299
Ben Widawsky07fe0b12013-07-31 17:00:10 -07004300 if (vma) {
Ben Widawskyd7f46fc2013-12-06 14:10:55 -08004301 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
4302 return -EBUSY;
4303
Chris Wilsond23db882014-05-23 08:48:08 +02004304 if (i915_vma_misplaced(vma, alignment, flags)) {
Ben Widawskyd7f46fc2013-12-06 14:10:55 -08004305 WARN(vma->pin_count,
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004306 "bo is already pinned in %s with incorrect alignment:"
Michel Thierry088e0df2015-08-07 17:40:17 +01004307 " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
Daniel Vetter75e9e912010-11-04 17:11:09 +01004308 " obj->map_and_fenceable=%d\n",
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004309 ggtt_view ? "ggtt" : "ppgtt",
Michel Thierry088e0df2015-08-07 17:40:17 +01004310 upper_32_bits(vma->node.start),
4311 lower_32_bits(vma->node.start),
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00004312 alignment,
Chris Wilsond23db882014-05-23 08:48:08 +02004313 !!(flags & PIN_MAPPABLE),
Chris Wilson05394f32010-11-08 19:18:58 +00004314 obj->map_and_fenceable);
Ben Widawsky07fe0b12013-07-31 17:00:10 -07004315 ret = i915_vma_unbind(vma);
Chris Wilsonac0c6b52010-05-27 13:18:18 +01004316 if (ret)
4317 return ret;
Daniel Vetter8ea99c92014-02-14 14:01:21 +01004318
4319 vma = NULL;
Chris Wilsonac0c6b52010-05-27 13:18:18 +01004320 }
4321 }
4322
Chris Wilsonef79e172014-10-31 13:53:52 +00004323 bound = vma ? vma->bound : 0;
Daniel Vetter8ea99c92014-02-14 14:01:21 +01004324 if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004325 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
4326 flags);
Daniel Vetter262de142014-02-14 14:01:20 +01004327 if (IS_ERR(vma))
4328 return PTR_ERR(vma);
Daniel Vetter08755462015-04-20 09:04:05 -07004329 } else {
4330 ret = i915_vma_bind(vma, obj->cache_level, flags);
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00004331 if (ret)
4332 return ret;
4333 }
Daniel Vetter74898d72012-02-15 23:50:22 +01004334
Joonas Lahtinen91e67112015-05-06 14:33:58 +03004335 if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
4336 (bound ^ vma->bound) & GLOBAL_BIND) {
Chris Wilsond0710ab2015-11-20 14:16:39 +00004337 __i915_vma_set_map_and_fenceable(vma);
Joonas Lahtinen91e67112015-05-06 14:33:58 +03004338 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
4339 }
Chris Wilsonef79e172014-10-31 13:53:52 +00004340
Daniel Vetter8ea99c92014-02-14 14:01:21 +01004341 vma->pin_count++;
Eric Anholt673a3942008-07-30 12:06:12 -07004342 return 0;
4343}
4344
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004345int
4346i915_gem_object_pin(struct drm_i915_gem_object *obj,
4347 struct i915_address_space *vm,
4348 uint32_t alignment,
4349 uint64_t flags)
4350{
4351 return i915_gem_object_do_pin(obj, vm,
4352 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
4353 alignment, flags);
4354}
4355
4356int
4357i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
4358 const struct i915_ggtt_view *view,
4359 uint32_t alignment,
4360 uint64_t flags)
4361{
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03004362 struct drm_device *dev = obj->base.dev;
4363 struct drm_i915_private *dev_priv = to_i915(dev);
4364 struct i915_ggtt *ggtt = &dev_priv->ggtt;
4365
Matthew Auldade7daa2016-03-24 15:54:20 +00004366 BUG_ON(!view);
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004367
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03004368 return i915_gem_object_do_pin(obj, &ggtt->base, view,
Tvrtko Ursulin6fafab72015-03-17 15:36:51 +00004369 alignment, flags | PIN_GLOBAL);
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004370}
4371
Eric Anholt673a3942008-07-30 12:06:12 -07004372void
Tvrtko Ursuline6617332015-03-23 11:10:33 +00004373i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
4374 const struct i915_ggtt_view *view)
Eric Anholt673a3942008-07-30 12:06:12 -07004375{
Tvrtko Ursuline6617332015-03-23 11:10:33 +00004376 struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
Eric Anholt673a3942008-07-30 12:06:12 -07004377
Ben Widawskyd7f46fc2013-12-06 14:10:55 -08004378 BUG_ON(!vma);
Tvrtko Ursuline6617332015-03-23 11:10:33 +00004379 WARN_ON(vma->pin_count == 0);
Joonas Lahtinen9abc4642015-03-27 13:09:22 +02004380 WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
Ben Widawskyd7f46fc2013-12-06 14:10:55 -08004381
Chris Wilson30154652015-04-07 17:28:24 +01004382 --vma->pin_count;
Eric Anholt673a3942008-07-30 12:06:12 -07004383}
4384
4385int
Eric Anholt673a3942008-07-30 12:06:12 -07004386i915_gem_busy_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00004387 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07004388{
4389 struct drm_i915_gem_busy *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00004390 struct drm_i915_gem_object *obj;
Chris Wilson30dbf0c2010-09-25 10:19:17 +01004391 int ret;
4392
Chris Wilson1d7cfea2010-10-17 09:45:41 +01004393 ret = i915_mutex_lock_interruptible(dev);
4394 if (ret)
4395 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004396
Chris Wilson05394f32010-11-08 19:18:58 +00004397 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00004398 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01004399 ret = -ENOENT;
4400 goto unlock;
Eric Anholt673a3942008-07-30 12:06:12 -07004401 }
Zou Nan haid1b851f2010-05-21 09:08:57 +08004402
Chris Wilson0be555b2010-08-04 15:36:30 +01004403 /* Count all active objects as busy, even if they are currently not used
4404 * by the gpu. Users of this interface expect objects to eventually
4405 * become non-busy without any further actions, therefore emit any
4406 * necessary flushes here.
Eric Anholtc4de0a52008-12-14 19:05:04 -08004407 */
Daniel Vetter30dfebf2012-06-01 15:21:23 +02004408 ret = i915_gem_object_flush_active(obj);
Chris Wilsonb4716182015-04-27 13:41:17 +01004409 if (ret)
4410 goto unref;
Daniel Vetter30dfebf2012-06-01 15:21:23 +02004411
Chris Wilson426960b2016-01-15 16:51:46 +00004412 args->busy = 0;
4413 if (obj->active) {
4414 int i;
4415
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00004416 for (i = 0; i < I915_NUM_ENGINES; i++) {
Chris Wilson426960b2016-01-15 16:51:46 +00004417 struct drm_i915_gem_request *req;
4418
4419 req = obj->last_read_req[i];
4420 if (req)
Tvrtko Ursulin4a570db2016-03-16 11:00:38 +00004421 args->busy |= 1 << (16 + req->engine->exec_id);
Chris Wilson426960b2016-01-15 16:51:46 +00004422 }
4423 if (obj->last_write_req)
Tvrtko Ursulin4a570db2016-03-16 11:00:38 +00004424 args->busy |= obj->last_write_req->engine->exec_id;
Chris Wilson426960b2016-01-15 16:51:46 +00004425 }
Eric Anholt673a3942008-07-30 12:06:12 -07004426
Chris Wilsonb4716182015-04-27 13:41:17 +01004427unref:
Chris Wilson05394f32010-11-08 19:18:58 +00004428 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01004429unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07004430 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01004431 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004432}
4433
4434int
4435i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4436 struct drm_file *file_priv)
4437{
Akshay Joshi0206e352011-08-16 15:34:10 -04004438 return i915_gem_ring_throttle(dev, file_priv);
Eric Anholt673a3942008-07-30 12:06:12 -07004439}
4440
Chris Wilson3ef94da2009-09-14 16:50:29 +01004441int
4442i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4443 struct drm_file *file_priv)
4444{
Daniel Vetter656bfa32014-11-20 09:26:30 +01004445 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004446 struct drm_i915_gem_madvise *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00004447 struct drm_i915_gem_object *obj;
Chris Wilson76c1dec2010-09-25 11:22:51 +01004448 int ret;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004449
4450 switch (args->madv) {
4451 case I915_MADV_DONTNEED:
4452 case I915_MADV_WILLNEED:
4453 break;
4454 default:
4455 return -EINVAL;
4456 }
4457
Chris Wilson1d7cfea2010-10-17 09:45:41 +01004458 ret = i915_mutex_lock_interruptible(dev);
4459 if (ret)
4460 return ret;
4461
Chris Wilson05394f32010-11-08 19:18:58 +00004462 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00004463 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01004464 ret = -ENOENT;
4465 goto unlock;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004466 }
Chris Wilson3ef94da2009-09-14 16:50:29 +01004467
Ben Widawskyd7f46fc2013-12-06 14:10:55 -08004468 if (i915_gem_obj_is_pinned(obj)) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01004469 ret = -EINVAL;
4470 goto out;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004471 }
4472
Daniel Vetter656bfa32014-11-20 09:26:30 +01004473 if (obj->pages &&
4474 obj->tiling_mode != I915_TILING_NONE &&
4475 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4476 if (obj->madv == I915_MADV_WILLNEED)
4477 i915_gem_object_unpin_pages(obj);
4478 if (args->madv == I915_MADV_WILLNEED)
4479 i915_gem_object_pin_pages(obj);
4480 }
4481
Chris Wilson05394f32010-11-08 19:18:58 +00004482 if (obj->madv != __I915_MADV_PURGED)
4483 obj->madv = args->madv;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004484
Chris Wilson6c085a72012-08-20 11:40:46 +02004485 /* if the object is no longer attached, discard its backing storage */
Daniel Vetterbe6a0372015-03-18 10:46:04 +01004486 if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
Chris Wilson2d7ef392009-09-20 23:13:10 +01004487 i915_gem_object_truncate(obj);
4488
Chris Wilson05394f32010-11-08 19:18:58 +00004489 args->retained = obj->madv != __I915_MADV_PURGED;
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004490
Chris Wilson1d7cfea2010-10-17 09:45:41 +01004491out:
Chris Wilson05394f32010-11-08 19:18:58 +00004492 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01004493unlock:
Chris Wilson3ef94da2009-09-14 16:50:29 +01004494 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01004495 return ret;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004496}
4497
Chris Wilson37e680a2012-06-07 15:38:42 +01004498void i915_gem_object_init(struct drm_i915_gem_object *obj,
4499 const struct drm_i915_gem_object_ops *ops)
Chris Wilson0327d6b2012-08-11 15:41:06 +01004500{
Chris Wilsonb4716182015-04-27 13:41:17 +01004501 int i;
4502
Ben Widawsky35c20a62013-05-31 11:28:48 -07004503 INIT_LIST_HEAD(&obj->global_list);
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00004504 for (i = 0; i < I915_NUM_ENGINES; i++)
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00004505 INIT_LIST_HEAD(&obj->engine_list[i]);
Ben Widawskyb25cb2f2013-08-14 11:38:33 +02004506 INIT_LIST_HEAD(&obj->obj_exec_link);
Ben Widawsky2f633152013-07-17 12:19:03 -07004507 INIT_LIST_HEAD(&obj->vma_list);
Chris Wilson8d9d5742015-04-07 16:20:38 +01004508 INIT_LIST_HEAD(&obj->batch_pool_link);
Chris Wilson0327d6b2012-08-11 15:41:06 +01004509
Chris Wilson37e680a2012-06-07 15:38:42 +01004510 obj->ops = ops;
4511
Chris Wilson0327d6b2012-08-11 15:41:06 +01004512 obj->fence_reg = I915_FENCE_REG_NONE;
4513 obj->madv = I915_MADV_WILLNEED;
Chris Wilson0327d6b2012-08-11 15:41:06 +01004514
4515 i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
4516}
4517
Chris Wilson37e680a2012-06-07 15:38:42 +01004518static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
Chris Wilsonde472662016-01-22 18:32:31 +00004519 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
Chris Wilson37e680a2012-06-07 15:38:42 +01004520 .get_pages = i915_gem_object_get_pages_gtt,
4521 .put_pages = i915_gem_object_put_pages_gtt,
4522};
4523
Chris Wilson05394f32010-11-08 19:18:58 +00004524struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4525 size_t size)
Daniel Vetterac52bc52010-04-09 19:05:06 +00004526{
Daniel Vetterc397b902010-04-09 19:05:07 +00004527 struct drm_i915_gem_object *obj;
Hugh Dickins5949eac2011-06-27 16:18:18 -07004528 struct address_space *mapping;
Daniel Vetter1a240d42012-11-29 22:18:51 +01004529 gfp_t mask;
Daniel Vetterc397b902010-04-09 19:05:07 +00004530
Chris Wilson42dcedd2012-11-15 11:32:30 +00004531 obj = i915_gem_object_alloc(dev);
Daniel Vetterc397b902010-04-09 19:05:07 +00004532 if (obj == NULL)
4533 return NULL;
4534
4535 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
Chris Wilson42dcedd2012-11-15 11:32:30 +00004536 i915_gem_object_free(obj);
Daniel Vetterc397b902010-04-09 19:05:07 +00004537 return NULL;
4538 }
4539
Chris Wilsonbed1ea92012-05-24 20:48:12 +01004540 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4541 if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4542 /* 965gm cannot relocate objects above 4GiB. */
4543 mask &= ~__GFP_HIGHMEM;
4544 mask |= __GFP_DMA32;
4545 }
4546
Al Viro496ad9a2013-01-23 17:07:38 -05004547 mapping = file_inode(obj->base.filp)->i_mapping;
Chris Wilsonbed1ea92012-05-24 20:48:12 +01004548 mapping_set_gfp_mask(mapping, mask);
Hugh Dickins5949eac2011-06-27 16:18:18 -07004549
Chris Wilson37e680a2012-06-07 15:38:42 +01004550 i915_gem_object_init(obj, &i915_gem_object_ops);
Chris Wilson73aa8082010-09-30 11:46:12 +01004551
Daniel Vetterc397b902010-04-09 19:05:07 +00004552 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4553 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4554
Eugeni Dodonov3d29b842012-01-17 14:43:53 -02004555 if (HAS_LLC(dev)) {
4556 /* On some devices, we can have the GPU use the LLC (the CPU
Eric Anholta1871112011-03-29 16:59:55 -07004557 * cache) for about a 10% performance improvement
4558 * compared to uncached. Graphics requests other than
4559 * display scanout are coherent with the CPU in
4560 * accessing this cache. This means in this mode we
4561 * don't need to clflush on the CPU side, and on the
4562 * GPU side we only need to flush internal caches to
4563 * get data visible to the CPU.
4564 *
4565 * However, we maintain the display planes as UC, and so
4566 * need to rebind when first used as such.
4567 */
4568 obj->cache_level = I915_CACHE_LLC;
4569 } else
4570 obj->cache_level = I915_CACHE_NONE;
4571
Daniel Vetterd861e332013-07-24 23:25:03 +02004572 trace_i915_gem_object_create(obj);
4573
Chris Wilson05394f32010-11-08 19:18:58 +00004574 return obj;
Daniel Vetterac52bc52010-04-09 19:05:06 +00004575}
4576
Chris Wilson340fbd82014-05-22 09:16:52 +01004577static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4578{
4579 /* If we are the last user of the backing storage (be it shmemfs
4580 * pages or stolen etc), we know that the pages are going to be
4581 * immediately released. In this case, we can then skip copying
4582 * back the contents from the GPU.
4583 */
4584
4585 if (obj->madv != I915_MADV_WILLNEED)
4586 return false;
4587
4588 if (obj->base.filp == NULL)
4589 return true;
4590
4591 /* At first glance, this looks racy, but then again so would be
4592 * userspace racing mmap against close. However, the first external
4593 * reference to the filp can only be obtained through the
4594 * i915_gem_mmap_ioctl() which safeguards us against the user
4595 * acquiring such a reference whilst we are in the middle of
4596 * freeing the object.
4597 */
4598 return atomic_long_read(&obj->base.filp->f_count) == 1;
4599}
4600
Chris Wilson1488fc02012-04-24 15:47:31 +01004601void i915_gem_free_object(struct drm_gem_object *gem_obj)
Chris Wilsonbe726152010-07-23 23:18:50 +01004602{
Chris Wilson1488fc02012-04-24 15:47:31 +01004603 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
Chris Wilson05394f32010-11-08 19:18:58 +00004604 struct drm_device *dev = obj->base.dev;
Jani Nikula3e31c6c2014-03-31 14:27:16 +03004605 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawsky07fe0b12013-07-31 17:00:10 -07004606 struct i915_vma *vma, *next;
Chris Wilsonbe726152010-07-23 23:18:50 +01004607
Paulo Zanonif65c9162013-11-27 18:20:34 -02004608 intel_runtime_pm_get(dev_priv);
4609
Chris Wilson26e12f892011-03-20 11:20:19 +00004610 trace_i915_gem_object_destroy(obj);
4611
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00004612 list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
Ben Widawskyd7f46fc2013-12-06 14:10:55 -08004613 int ret;
4614
4615 vma->pin_count = 0;
4616 ret = i915_vma_unbind(vma);
Ben Widawsky07fe0b12013-07-31 17:00:10 -07004617 if (WARN_ON(ret == -ERESTARTSYS)) {
4618 bool was_interruptible;
Chris Wilson1488fc02012-04-24 15:47:31 +01004619
Ben Widawsky07fe0b12013-07-31 17:00:10 -07004620 was_interruptible = dev_priv->mm.interruptible;
4621 dev_priv->mm.interruptible = false;
Chris Wilson1488fc02012-04-24 15:47:31 +01004622
Ben Widawsky07fe0b12013-07-31 17:00:10 -07004623 WARN_ON(i915_vma_unbind(vma));
Chris Wilson1488fc02012-04-24 15:47:31 +01004624
Ben Widawsky07fe0b12013-07-31 17:00:10 -07004625 dev_priv->mm.interruptible = was_interruptible;
4626 }
Chris Wilson1488fc02012-04-24 15:47:31 +01004627 }
4628
Ben Widawsky1d64ae72013-05-31 14:46:20 -07004629 /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4630 * before progressing. */
4631 if (obj->stolen)
4632 i915_gem_object_unpin_pages(obj);
4633
Daniel Vettera071fa02014-06-18 23:28:09 +02004634 WARN_ON(obj->frontbuffer_bits);
4635
Daniel Vetter656bfa32014-11-20 09:26:30 +01004636 if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4637 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4638 obj->tiling_mode != I915_TILING_NONE)
4639 i915_gem_object_unpin_pages(obj);
4640
Ben Widawsky401c29f2013-05-31 11:28:47 -07004641 if (WARN_ON(obj->pages_pin_count))
4642 obj->pages_pin_count = 0;
Chris Wilson340fbd82014-05-22 09:16:52 +01004643 if (discard_backing_storage(obj))
Chris Wilson55372522014-03-25 13:23:06 +00004644 obj->madv = I915_MADV_DONTNEED;
Chris Wilson37e680a2012-06-07 15:38:42 +01004645 i915_gem_object_put_pages(obj);
Chris Wilsond8cb5082012-08-11 15:41:03 +01004646 i915_gem_object_free_mmap_offset(obj);
Chris Wilsonbe726152010-07-23 23:18:50 +01004647
Chris Wilson9da3da62012-06-01 15:20:22 +01004648 BUG_ON(obj->pages);
4649
Chris Wilson2f745ad2012-09-04 21:02:58 +01004650 if (obj->base.import_attach)
4651 drm_prime_gem_destroy(&obj->base, NULL);
Chris Wilsonbe726152010-07-23 23:18:50 +01004652
Chris Wilson5cc9ed42014-05-16 14:22:37 +01004653 if (obj->ops->release)
4654 obj->ops->release(obj);
4655
Chris Wilson05394f32010-11-08 19:18:58 +00004656 drm_gem_object_release(&obj->base);
4657 i915_gem_info_remove_obj(dev_priv, obj->base.size);
Chris Wilsonbe726152010-07-23 23:18:50 +01004658
Chris Wilson05394f32010-11-08 19:18:58 +00004659 kfree(obj->bit_17);
Chris Wilson42dcedd2012-11-15 11:32:30 +00004660 i915_gem_object_free(obj);
Paulo Zanonif65c9162013-11-27 18:20:34 -02004661
4662 intel_runtime_pm_put(dev_priv);
Chris Wilsonbe726152010-07-23 23:18:50 +01004663}
4664
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004665struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4666 struct i915_address_space *vm)
Ben Widawsky2f633152013-07-17 12:19:03 -07004667{
Daniel Vettere656a6c2013-08-14 14:14:04 +02004668 struct i915_vma *vma;
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00004669 list_for_each_entry(vma, &obj->vma_list, obj_link) {
Tvrtko Ursulin1b683722015-11-12 11:59:55 +00004670 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
4671 vma->vm == vm)
Daniel Vettere656a6c2013-08-14 14:14:04 +02004672 return vma;
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004673 }
4674 return NULL;
4675}
Daniel Vettere656a6c2013-08-14 14:14:04 +02004676
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004677struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
4678 const struct i915_ggtt_view *view)
4679{
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03004680 struct drm_device *dev = obj->base.dev;
4681 struct drm_i915_private *dev_priv = to_i915(dev);
4682 struct i915_ggtt *ggtt = &dev_priv->ggtt;
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004683 struct i915_vma *vma;
4684
Matthew Auldade7daa2016-03-24 15:54:20 +00004685 BUG_ON(!view);
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004686
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00004687 list_for_each_entry(vma, &obj->vma_list, obj_link)
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03004688 if (vma->vm == &ggtt->base &&
Joonas Lahtinen9abc4642015-03-27 13:09:22 +02004689 i915_ggtt_view_equal(&vma->ggtt_view, view))
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004690 return vma;
Daniel Vettere656a6c2013-08-14 14:14:04 +02004691 return NULL;
4692}
4693
Ben Widawsky2f633152013-07-17 12:19:03 -07004694void i915_gem_vma_destroy(struct i915_vma *vma)
4695{
4696 WARN_ON(vma->node.allocated);
Chris Wilsonaaa05662013-08-20 12:56:40 +01004697
4698 /* Keep the vma as a placeholder in the execbuffer reservation lists */
4699 if (!list_empty(&vma->exec_list))
4700 return;
4701
Chris Wilson596c5922016-02-26 11:03:20 +00004702 if (!vma->is_ggtt)
4703 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
Michel Thierryb9d06dd2014-08-06 15:04:44 +02004704
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00004705 list_del(&vma->obj_link);
Daniel Vetterb93dab62013-08-26 11:23:47 +02004706
Chris Wilsone20d2ab2015-04-07 16:20:58 +01004707 kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
Ben Widawsky2f633152013-07-17 12:19:03 -07004708}
4709
Chris Wilsone3efda42014-04-09 09:19:41 +01004710static void
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00004711i915_gem_stop_engines(struct drm_device *dev)
Chris Wilsone3efda42014-04-09 09:19:41 +01004712{
4713 struct drm_i915_private *dev_priv = dev->dev_private;
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00004714 struct intel_engine_cs *engine;
Chris Wilsone3efda42014-04-09 09:19:41 +01004715
Dave Gordonb4ac5af2016-03-24 11:20:38 +00004716 for_each_engine(engine, dev_priv)
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00004717 dev_priv->gt.stop_engine(engine);
Chris Wilsone3efda42014-04-09 09:19:41 +01004718}
4719
Jesse Barnes5669fca2009-02-17 15:13:31 -08004720int
Chris Wilson45c5f202013-10-16 11:50:01 +01004721i915_gem_suspend(struct drm_device *dev)
Eric Anholt673a3942008-07-30 12:06:12 -07004722{
Jani Nikula3e31c6c2014-03-31 14:27:16 +03004723 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson45c5f202013-10-16 11:50:01 +01004724 int ret = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07004725
Chris Wilson45c5f202013-10-16 11:50:01 +01004726 mutex_lock(&dev->struct_mutex);
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07004727 ret = i915_gpu_idle(dev);
Chris Wilsonf7403342013-09-13 23:57:04 +01004728 if (ret)
Chris Wilson45c5f202013-10-16 11:50:01 +01004729 goto err;
Chris Wilsonf7403342013-09-13 23:57:04 +01004730
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07004731 i915_gem_retire_requests(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07004732
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00004733 i915_gem_stop_engines(dev);
Chris Wilson45c5f202013-10-16 11:50:01 +01004734 mutex_unlock(&dev->struct_mutex);
4735
Chris Wilson737b1502015-01-26 18:03:03 +02004736 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
Chris Wilson29105cc2010-01-07 10:39:13 +00004737 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
Deepak S274fa1c2014-08-05 07:51:20 -07004738 flush_delayed_work(&dev_priv->mm.idle_work);
Chris Wilson29105cc2010-01-07 10:39:13 +00004739
Chris Wilsonbdcf1202014-11-25 11:56:33 +00004740 /* Assert that we sucessfully flushed all the work and
4741 * reset the GPU back to its idle, low power state.
4742 */
4743 WARN_ON(dev_priv->mm.busy);
4744
Eric Anholt673a3942008-07-30 12:06:12 -07004745 return 0;
Chris Wilson45c5f202013-10-16 11:50:01 +01004746
4747err:
4748 mutex_unlock(&dev->struct_mutex);
4749 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004750}
4751
John Harrison6909a662015-05-29 17:43:51 +01004752int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
Ben Widawskyb9524a12012-05-25 16:56:24 -07004753{
Tvrtko Ursulin4a570db2016-03-16 11:00:38 +00004754 struct intel_engine_cs *engine = req->engine;
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00004755 struct drm_device *dev = engine->dev;
Jani Nikula3e31c6c2014-03-31 14:27:16 +03004756 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawsky35a85ac2013-09-19 11:13:41 -07004757 u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
Ben Widawskyc3787e22013-09-17 21:12:44 -07004758 int i, ret;
Ben Widawskyb9524a12012-05-25 16:56:24 -07004759
Ben Widawsky040d2ba2013-09-19 11:01:40 -07004760 if (!HAS_L3_DPF(dev) || !remap_info)
Ben Widawskyc3787e22013-09-17 21:12:44 -07004761 return 0;
Ben Widawskyb9524a12012-05-25 16:56:24 -07004762
John Harrison5fb9de12015-05-29 17:44:07 +01004763 ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3);
Ben Widawskyc3787e22013-09-17 21:12:44 -07004764 if (ret)
4765 return ret;
Ben Widawskyb9524a12012-05-25 16:56:24 -07004766
Ben Widawskyc3787e22013-09-17 21:12:44 -07004767 /*
4768 * Note: We do not worry about the concurrent register cacheline hang
4769 * here because no other code should access these registers other than
4770 * at initialization time.
4771 */
Ville Syrjälä6fa1c5f2015-11-04 23:20:02 +02004772 for (i = 0; i < GEN7_L3LOG_SIZE / 4; i++) {
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00004773 intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
4774 intel_ring_emit_reg(engine, GEN7_L3LOG(slice, i));
4775 intel_ring_emit(engine, remap_info[i]);
Ben Widawskyb9524a12012-05-25 16:56:24 -07004776 }
4777
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00004778 intel_ring_advance(engine);
Ben Widawskyb9524a12012-05-25 16:56:24 -07004779
Ben Widawskyc3787e22013-09-17 21:12:44 -07004780 return ret;
Ben Widawskyb9524a12012-05-25 16:56:24 -07004781}
4782
Daniel Vetterf691e2f2012-02-02 09:58:12 +01004783void i915_gem_init_swizzling(struct drm_device *dev)
4784{
Jani Nikula3e31c6c2014-03-31 14:27:16 +03004785 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetterf691e2f2012-02-02 09:58:12 +01004786
Daniel Vetter11782b02012-01-31 16:47:55 +01004787 if (INTEL_INFO(dev)->gen < 5 ||
Daniel Vetterf691e2f2012-02-02 09:58:12 +01004788 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4789 return;
4790
4791 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4792 DISP_TILE_SURFACE_SWIZZLING);
4793
Daniel Vetter11782b02012-01-31 16:47:55 +01004794 if (IS_GEN5(dev))
4795 return;
4796
Daniel Vetterf691e2f2012-02-02 09:58:12 +01004797 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4798 if (IS_GEN6(dev))
Daniel Vetter6b26c862012-04-24 14:04:12 +02004799 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
Ben Widawsky8782e262012-12-18 10:31:23 -08004800 else if (IS_GEN7(dev))
Daniel Vetter6b26c862012-04-24 14:04:12 +02004801 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
Ben Widawsky31a53362013-11-02 21:07:04 -07004802 else if (IS_GEN8(dev))
4803 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
Ben Widawsky8782e262012-12-18 10:31:23 -08004804 else
4805 BUG();
Daniel Vetterf691e2f2012-02-02 09:58:12 +01004806}
Daniel Vettere21af882012-02-09 20:53:27 +01004807
Ville Syrjälä81e7f202014-08-15 01:21:55 +03004808static void init_unused_ring(struct drm_device *dev, u32 base)
4809{
4810 struct drm_i915_private *dev_priv = dev->dev_private;
4811
4812 I915_WRITE(RING_CTL(base), 0);
4813 I915_WRITE(RING_HEAD(base), 0);
4814 I915_WRITE(RING_TAIL(base), 0);
4815 I915_WRITE(RING_START(base), 0);
4816}
4817
4818static void init_unused_rings(struct drm_device *dev)
4819{
4820 if (IS_I830(dev)) {
4821 init_unused_ring(dev, PRB1_BASE);
4822 init_unused_ring(dev, SRB0_BASE);
4823 init_unused_ring(dev, SRB1_BASE);
4824 init_unused_ring(dev, SRB2_BASE);
4825 init_unused_ring(dev, SRB3_BASE);
4826 } else if (IS_GEN2(dev)) {
4827 init_unused_ring(dev, SRB0_BASE);
4828 init_unused_ring(dev, SRB1_BASE);
4829 } else if (IS_GEN3(dev)) {
4830 init_unused_ring(dev, PRB1_BASE);
4831 init_unused_ring(dev, PRB2_BASE);
4832 }
4833}
4834
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00004835int i915_gem_init_engines(struct drm_device *dev)
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004836{
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004837 struct drm_i915_private *dev_priv = dev->dev_private;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004838 int ret;
Chris Wilson68f95ba2010-05-27 13:18:22 +01004839
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08004840 ret = intel_init_render_ring_buffer(dev);
Chris Wilson68f95ba2010-05-27 13:18:22 +01004841 if (ret)
Chris Wilsonb6913e42010-11-12 10:46:37 +00004842 return ret;
Chris Wilson68f95ba2010-05-27 13:18:22 +01004843
4844 if (HAS_BSD(dev)) {
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08004845 ret = intel_init_bsd_ring_buffer(dev);
Chris Wilson68f95ba2010-05-27 13:18:22 +01004846 if (ret)
4847 goto cleanup_render_ring;
Zou Nan haid1b851f2010-05-21 09:08:57 +08004848 }
Chris Wilson68f95ba2010-05-27 13:18:22 +01004849
Jani Nikulad39398f2015-10-07 11:17:44 +03004850 if (HAS_BLT(dev)) {
Chris Wilson549f7362010-10-19 11:19:32 +01004851 ret = intel_init_blt_ring_buffer(dev);
4852 if (ret)
4853 goto cleanup_bsd_ring;
4854 }
4855
Ben Widawsky9a8a2212013-05-28 19:22:23 -07004856 if (HAS_VEBOX(dev)) {
4857 ret = intel_init_vebox_ring_buffer(dev);
4858 if (ret)
4859 goto cleanup_blt_ring;
4860 }
4861
Zhao Yakui845f74a2014-04-17 10:37:37 +08004862 if (HAS_BSD2(dev)) {
4863 ret = intel_init_bsd2_ring_buffer(dev);
4864 if (ret)
4865 goto cleanup_vebox_ring;
4866 }
Ben Widawsky9a8a2212013-05-28 19:22:23 -07004867
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004868 return 0;
4869
Ben Widawsky9a8a2212013-05-28 19:22:23 -07004870cleanup_vebox_ring:
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00004871 intel_cleanup_engine(&dev_priv->engine[VECS]);
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004872cleanup_blt_ring:
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00004873 intel_cleanup_engine(&dev_priv->engine[BCS]);
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004874cleanup_bsd_ring:
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00004875 intel_cleanup_engine(&dev_priv->engine[VCS]);
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004876cleanup_render_ring:
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00004877 intel_cleanup_engine(&dev_priv->engine[RCS]);
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004878
4879 return ret;
4880}
4881
4882int
4883i915_gem_init_hw(struct drm_device *dev)
4884{
Jani Nikula3e31c6c2014-03-31 14:27:16 +03004885 struct drm_i915_private *dev_priv = dev->dev_private;
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00004886 struct intel_engine_cs *engine;
Dave Gordonb4ac5af2016-03-24 11:20:38 +00004887 int ret, j;
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004888
4889 if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
4890 return -EIO;
4891
Chris Wilson5e4f5182015-02-13 14:35:59 +00004892 /* Double layer security blanket, see i915_gem_init() */
4893 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4894
Mika Kuoppala3accaf72016-04-13 17:26:43 +03004895 if (HAS_EDRAM(dev) && INTEL_GEN(dev_priv) < 9)
Ben Widawsky05e21cc2013-07-04 11:02:04 -07004896 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004897
Ville Syrjälä0bf21342013-11-29 14:56:12 +02004898 if (IS_HASWELL(dev))
4899 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4900 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
Rodrigo Vivi94353732013-08-28 16:45:46 -03004901
Ben Widawsky88a2b2a2013-04-05 13:12:43 -07004902 if (HAS_PCH_NOP(dev)) {
Daniel Vetter6ba844b2014-01-22 23:39:30 +01004903 if (IS_IVYBRIDGE(dev)) {
4904 u32 temp = I915_READ(GEN7_MSG_CTL);
4905 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4906 I915_WRITE(GEN7_MSG_CTL, temp);
4907 } else if (INTEL_INFO(dev)->gen >= 7) {
4908 u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4909 temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4910 I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4911 }
Ben Widawsky88a2b2a2013-04-05 13:12:43 -07004912 }
4913
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004914 i915_gem_init_swizzling(dev);
4915
Daniel Vetterd5abdfd2014-11-20 09:45:19 +01004916 /*
4917 * At least 830 can leave some of the unused rings
4918 * "active" (ie. head != tail) after resume which
4919 * will prevent c3 entry. Makes sure all unused rings
4920 * are totally idle.
4921 */
4922 init_unused_rings(dev);
4923
Dave Gordoned54c1a2016-01-19 19:02:54 +00004924 BUG_ON(!dev_priv->kernel_context);
John Harrison90638cc2015-05-29 17:43:37 +01004925
John Harrison4ad2fd82015-06-18 13:11:20 +01004926 ret = i915_ppgtt_init_hw(dev);
4927 if (ret) {
4928 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
4929 goto out;
4930 }
4931
4932 /* Need to do basic initialisation of all rings first: */
Dave Gordonb4ac5af2016-03-24 11:20:38 +00004933 for_each_engine(engine, dev_priv) {
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00004934 ret = engine->init_hw(engine);
Daniel Vetter35a57ff2014-11-20 00:33:07 +01004935 if (ret)
Chris Wilson5e4f5182015-02-13 14:35:59 +00004936 goto out;
Daniel Vetter35a57ff2014-11-20 00:33:07 +01004937 }
Mika Kuoppala99433932013-01-22 14:12:17 +02004938
Alex Dai33a732f2015-08-12 15:43:36 +01004939 /* We can't enable contexts until all firmware is loaded */
Jesse Barnes87bcdd22015-09-10 14:55:00 -07004940 if (HAS_GUC_UCODE(dev)) {
4941 ret = intel_guc_ucode_load(dev);
4942 if (ret) {
Daniel Vetter9f9e5392015-10-23 11:10:59 +02004943 DRM_ERROR("Failed to initialize GuC, error %d\n", ret);
4944 ret = -EIO;
4945 goto out;
Jesse Barnes87bcdd22015-09-10 14:55:00 -07004946 }
Alex Dai33a732f2015-08-12 15:43:36 +01004947 }
4948
Nick Hoathe84fe802015-09-11 12:53:46 +01004949 /*
4950 * Increment the next seqno by 0x100 so we have a visible break
4951 * on re-initialisation
4952 */
4953 ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
4954 if (ret)
4955 goto out;
4956
John Harrison4ad2fd82015-06-18 13:11:20 +01004957 /* Now it is safe to go back round and do everything else: */
Dave Gordonb4ac5af2016-03-24 11:20:38 +00004958 for_each_engine(engine, dev_priv) {
John Harrisondc4be60712015-05-29 17:43:39 +01004959 struct drm_i915_gem_request *req;
4960
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00004961 req = i915_gem_request_alloc(engine, NULL);
Dave Gordon26827082016-01-19 19:02:53 +00004962 if (IS_ERR(req)) {
4963 ret = PTR_ERR(req);
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00004964 i915_gem_cleanup_engines(dev);
John Harrisondc4be60712015-05-29 17:43:39 +01004965 goto out;
4966 }
4967
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00004968 if (engine->id == RCS) {
John Harrison4ad2fd82015-06-18 13:11:20 +01004969 for (j = 0; j < NUM_L3_SLICES(dev); j++)
John Harrison6909a662015-05-29 17:43:51 +01004970 i915_gem_l3_remap(req, j);
John Harrison4ad2fd82015-06-18 13:11:20 +01004971 }
Ben Widawskyc3787e22013-09-17 21:12:44 -07004972
John Harrisonb3dd6b92015-05-29 17:43:40 +01004973 ret = i915_ppgtt_init_ring(req);
John Harrison4ad2fd82015-06-18 13:11:20 +01004974 if (ret && ret != -EIO) {
Dave Gordonb4ac5af2016-03-24 11:20:38 +00004975 DRM_ERROR("PPGTT enable %s failed %d\n",
4976 engine->name, ret);
John Harrisondc4be60712015-05-29 17:43:39 +01004977 i915_gem_request_cancel(req);
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00004978 i915_gem_cleanup_engines(dev);
John Harrison4ad2fd82015-06-18 13:11:20 +01004979 goto out;
4980 }
David Woodhousef48a0162015-01-20 17:21:42 +00004981
John Harrisonb3dd6b92015-05-29 17:43:40 +01004982 ret = i915_gem_context_enable(req);
John Harrison90638cc2015-05-29 17:43:37 +01004983 if (ret && ret != -EIO) {
Dave Gordonb4ac5af2016-03-24 11:20:38 +00004984 DRM_ERROR("Context enable %s failed %d\n",
4985 engine->name, ret);
John Harrisondc4be60712015-05-29 17:43:39 +01004986 i915_gem_request_cancel(req);
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00004987 i915_gem_cleanup_engines(dev);
John Harrison90638cc2015-05-29 17:43:37 +01004988 goto out;
4989 }
John Harrisondc4be60712015-05-29 17:43:39 +01004990
John Harrison75289872015-05-29 17:43:49 +01004991 i915_add_request_no_flush(req);
Daniel Vetter82460d92014-08-06 20:19:53 +02004992 }
4993
Chris Wilson5e4f5182015-02-13 14:35:59 +00004994out:
4995 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
Ben Widawsky2fa48d82013-12-06 14:11:04 -08004996 return ret;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004997}
4998
Chris Wilson1070a422012-04-24 15:47:41 +01004999int i915_gem_init(struct drm_device *dev)
5000{
5001 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson1070a422012-04-24 15:47:41 +01005002 int ret;
5003
Oscar Mateo127f1002014-07-24 17:04:11 +01005004 i915.enable_execlists = intel_sanitize_enable_execlists(dev,
5005 i915.enable_execlists);
5006
Chris Wilson1070a422012-04-24 15:47:41 +01005007 mutex_lock(&dev->struct_mutex);
Jesse Barnesd62b4892013-03-08 10:45:53 -08005008
Oscar Mateoa83014d2014-07-24 17:04:21 +01005009 if (!i915.enable_execlists) {
John Harrisonf3dc74c2015-03-19 12:30:06 +00005010 dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00005011 dev_priv->gt.init_engines = i915_gem_init_engines;
5012 dev_priv->gt.cleanup_engine = intel_cleanup_engine;
5013 dev_priv->gt.stop_engine = intel_stop_engine;
Oscar Mateo454afeb2014-07-24 17:04:22 +01005014 } else {
John Harrisonf3dc74c2015-03-19 12:30:06 +00005015 dev_priv->gt.execbuf_submit = intel_execlists_submission;
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00005016 dev_priv->gt.init_engines = intel_logical_rings_init;
5017 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
5018 dev_priv->gt.stop_engine = intel_logical_ring_stop;
Oscar Mateoa83014d2014-07-24 17:04:21 +01005019 }
5020
Chris Wilson5e4f5182015-02-13 14:35:59 +00005021 /* This is just a security blanket to placate dragons.
5022 * On some systems, we very sporadically observe that the first TLBs
5023 * used by the CS may be stale, despite us poking the TLB reset. If
5024 * we hold the forcewake during initialisation these problems
5025 * just magically go away.
5026 */
5027 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5028
Daniel Vetter6c5566a2014-08-06 15:04:50 +02005029 ret = i915_gem_init_userptr(dev);
Jani Nikula7bcc3772014-12-05 14:17:42 +02005030 if (ret)
5031 goto out_unlock;
Daniel Vetter6c5566a2014-08-06 15:04:50 +02005032
Joonas Lahtinend85489d2016-03-24 16:47:46 +02005033 i915_gem_init_ggtt(dev);
Jesse Barnesd62b4892013-03-08 10:45:53 -08005034
Ben Widawsky2fa48d82013-12-06 14:11:04 -08005035 ret = i915_gem_context_init(dev);
Jani Nikula7bcc3772014-12-05 14:17:42 +02005036 if (ret)
5037 goto out_unlock;
Ben Widawsky2fa48d82013-12-06 14:11:04 -08005038
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00005039 ret = dev_priv->gt.init_engines(dev);
Daniel Vetter35a57ff2014-11-20 00:33:07 +01005040 if (ret)
Jani Nikula7bcc3772014-12-05 14:17:42 +02005041 goto out_unlock;
Daniel Vetter53ca26c2012-04-26 23:28:03 +02005042
5043 ret = i915_gem_init_hw(dev);
Chris Wilson60990322014-04-09 09:19:42 +01005044 if (ret == -EIO) {
5045 /* Allow ring initialisation to fail by marking the GPU as
5046 * wedged. But we only want to do this where the GPU is angry,
5047 * for all other failure, such as an allocation failure, bail.
5048 */
5049 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
Peter Zijlstra805de8f42015-04-24 01:12:32 +02005050 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
Chris Wilson60990322014-04-09 09:19:42 +01005051 ret = 0;
Chris Wilson1070a422012-04-24 15:47:41 +01005052 }
Jani Nikula7bcc3772014-12-05 14:17:42 +02005053
5054out_unlock:
Chris Wilson5e4f5182015-02-13 14:35:59 +00005055 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
Chris Wilson60990322014-04-09 09:19:42 +01005056 mutex_unlock(&dev->struct_mutex);
Chris Wilson1070a422012-04-24 15:47:41 +01005057
Chris Wilson60990322014-04-09 09:19:42 +01005058 return ret;
Chris Wilson1070a422012-04-24 15:47:41 +01005059}
5060
Zou Nan hai8187a2b2010-05-21 09:08:55 +08005061void
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00005062i915_gem_cleanup_engines(struct drm_device *dev)
Zou Nan hai8187a2b2010-05-21 09:08:55 +08005063{
Jani Nikula3e31c6c2014-03-31 14:27:16 +03005064 struct drm_i915_private *dev_priv = dev->dev_private;
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00005065 struct intel_engine_cs *engine;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08005066
Dave Gordonb4ac5af2016-03-24 11:20:38 +00005067 for_each_engine(engine, dev_priv)
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00005068 dev_priv->gt.cleanup_engine(engine);
Niu,Binga6478282015-07-04 00:27:34 +08005069
Mika Kuoppalaee4b6fa2016-03-16 17:54:00 +02005070 if (i915.enable_execlists)
5071 /*
5072 * Neither the BIOS, ourselves or any other kernel
5073 * expects the system to be in execlists mode on startup,
5074 * so we need to reset the GPU back to legacy mode.
5075 */
5076 intel_gpu_reset(dev, ALL_ENGINES);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08005077}
5078
Chris Wilson64193402010-10-24 12:38:05 +01005079static void
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00005080init_engine_lists(struct intel_engine_cs *engine)
Chris Wilson64193402010-10-24 12:38:05 +01005081{
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00005082 INIT_LIST_HEAD(&engine->active_list);
5083 INIT_LIST_HEAD(&engine->request_list);
Chris Wilson64193402010-10-24 12:38:05 +01005084}
5085
Eric Anholt673a3942008-07-30 12:06:12 -07005086void
Imre Deak40ae4e12016-03-16 14:54:03 +02005087i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
5088{
5089 struct drm_device *dev = dev_priv->dev;
5090
5091 if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
5092 !IS_CHERRYVIEW(dev_priv))
5093 dev_priv->num_fence_regs = 32;
5094 else if (INTEL_INFO(dev_priv)->gen >= 4 || IS_I945G(dev_priv) ||
5095 IS_I945GM(dev_priv) || IS_G33(dev_priv))
5096 dev_priv->num_fence_regs = 16;
5097 else
5098 dev_priv->num_fence_regs = 8;
5099
5100 if (intel_vgpu_active(dev))
5101 dev_priv->num_fence_regs =
5102 I915_READ(vgtif_reg(avail_rs.fence_num));
5103
5104 /* Initialize fence registers to zero */
5105 i915_gem_restore_fences(dev);
5106
5107 i915_gem_detect_bit_6_swizzle(dev);
5108}
5109
5110void
Imre Deakd64aa092016-01-19 15:26:29 +02005111i915_gem_load_init(struct drm_device *dev)
Eric Anholt673a3942008-07-30 12:06:12 -07005112{
Jani Nikula3e31c6c2014-03-31 14:27:16 +03005113 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson42dcedd2012-11-15 11:32:30 +00005114 int i;
5115
Chris Wilsonefab6d82015-04-07 16:20:57 +01005116 dev_priv->objects =
Chris Wilson42dcedd2012-11-15 11:32:30 +00005117 kmem_cache_create("i915_gem_object",
5118 sizeof(struct drm_i915_gem_object), 0,
5119 SLAB_HWCACHE_ALIGN,
5120 NULL);
Chris Wilsone20d2ab2015-04-07 16:20:58 +01005121 dev_priv->vmas =
5122 kmem_cache_create("i915_gem_vma",
5123 sizeof(struct i915_vma), 0,
5124 SLAB_HWCACHE_ALIGN,
5125 NULL);
Chris Wilsonefab6d82015-04-07 16:20:57 +01005126 dev_priv->requests =
5127 kmem_cache_create("i915_gem_request",
5128 sizeof(struct drm_i915_gem_request), 0,
5129 SLAB_HWCACHE_ALIGN,
5130 NULL);
Eric Anholt673a3942008-07-30 12:06:12 -07005131
Ben Widawskyfc8c0672013-07-31 16:59:54 -07005132 INIT_LIST_HEAD(&dev_priv->vm_list);
Ben Widawskya33afea2013-09-17 21:12:45 -07005133 INIT_LIST_HEAD(&dev_priv->context_list);
Chris Wilson6c085a72012-08-20 11:40:46 +02005134 INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
5135 INIT_LIST_HEAD(&dev_priv->mm.bound_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07005136 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
Tvrtko Ursulin666796d2016-03-16 11:00:39 +00005137 for (i = 0; i < I915_NUM_ENGINES; i++)
5138 init_engine_lists(&dev_priv->engine[i]);
Daniel Vetter4b9de732011-10-09 21:52:02 +02005139 for (i = 0; i < I915_MAX_NUM_FENCES; i++)
Daniel Vetter007cc8a2010-04-28 11:02:31 +02005140 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
Eric Anholt673a3942008-07-30 12:06:12 -07005141 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
5142 i915_gem_retire_work_handler);
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005143 INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
5144 i915_gem_idle_work_handler);
Daniel Vetter1f83fee2012-11-15 17:17:22 +01005145 init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
Chris Wilson31169712009-09-14 16:50:28 +01005146
Chris Wilson72bfa192010-12-19 11:42:05 +00005147 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
5148
Nick Hoathe84fe802015-09-11 12:53:46 +01005149 /*
5150 * Set initial sequence number for requests.
5151 * Using this number allows the wraparound to happen early,
5152 * catching any obvious problems.
5153 */
5154 dev_priv->next_seqno = ((u32)~0 - 0x1100);
5155 dev_priv->last_seqno = ((u32)~0 - 0x1101);
5156
Chris Wilson19b2dbd2013-06-12 10:15:12 +01005157 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
Eric Anholt10ed13e2011-05-06 13:53:49 -07005158
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05005159 init_waitqueue_head(&dev_priv->pending_flip_queue);
Chris Wilson17250b72010-10-28 12:51:39 +01005160
Chris Wilsonce453d82011-02-21 14:43:56 +00005161 dev_priv->mm.interruptible = true;
5162
Daniel Vetterf99d7062014-06-19 16:01:59 +02005163 mutex_init(&dev_priv->fb_tracking.lock);
Eric Anholt673a3942008-07-30 12:06:12 -07005164}
Dave Airlie71acb5e2008-12-30 20:31:46 +10005165
Imre Deakd64aa092016-01-19 15:26:29 +02005166void i915_gem_load_cleanup(struct drm_device *dev)
5167{
5168 struct drm_i915_private *dev_priv = to_i915(dev);
5169
5170 kmem_cache_destroy(dev_priv->requests);
5171 kmem_cache_destroy(dev_priv->vmas);
5172 kmem_cache_destroy(dev_priv->objects);
5173}
5174
Chris Wilsonf787a5f2010-09-24 16:02:42 +01005175void i915_gem_release(struct drm_device *dev, struct drm_file *file)
Eric Anholtb9624422009-06-03 07:27:35 +00005176{
Chris Wilsonf787a5f2010-09-24 16:02:42 +01005177 struct drm_i915_file_private *file_priv = file->driver_priv;
Eric Anholtb9624422009-06-03 07:27:35 +00005178
5179 /* Clean up our request list when the client is going away, so that
5180 * later retire_requests won't dereference our soon-to-be-gone
5181 * file_priv.
5182 */
Chris Wilson1c255952010-09-26 11:03:27 +01005183 spin_lock(&file_priv->mm.lock);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01005184 while (!list_empty(&file_priv->mm.request_list)) {
5185 struct drm_i915_gem_request *request;
5186
5187 request = list_first_entry(&file_priv->mm.request_list,
5188 struct drm_i915_gem_request,
5189 client_list);
5190 list_del(&request->client_list);
5191 request->file_priv = NULL;
5192 }
Chris Wilson1c255952010-09-26 11:03:27 +01005193 spin_unlock(&file_priv->mm.lock);
Chris Wilson31169712009-09-14 16:50:28 +01005194
Chris Wilson2e1b8732015-04-27 13:41:22 +01005195 if (!list_empty(&file_priv->rps.link)) {
Chris Wilson8d3afd72015-05-21 21:01:47 +01005196 spin_lock(&to_i915(dev)->rps.client_lock);
Chris Wilson2e1b8732015-04-27 13:41:22 +01005197 list_del(&file_priv->rps.link);
Chris Wilson8d3afd72015-05-21 21:01:47 +01005198 spin_unlock(&to_i915(dev)->rps.client_lock);
Chris Wilson1854d5c2015-04-07 16:20:32 +01005199 }
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005200}
5201
5202int i915_gem_open(struct drm_device *dev, struct drm_file *file)
5203{
5204 struct drm_i915_file_private *file_priv;
Ben Widawskye422b882013-12-06 14:10:58 -08005205 int ret;
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005206
5207 DRM_DEBUG_DRIVER("\n");
5208
5209 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
5210 if (!file_priv)
5211 return -ENOMEM;
5212
5213 file->driver_priv = file_priv;
5214 file_priv->dev_priv = dev->dev_private;
Chris Wilsonab0e7ff2014-02-25 17:11:24 +02005215 file_priv->file = file;
Chris Wilson2e1b8732015-04-27 13:41:22 +01005216 INIT_LIST_HEAD(&file_priv->rps.link);
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005217
5218 spin_lock_init(&file_priv->mm.lock);
5219 INIT_LIST_HEAD(&file_priv->mm.request_list);
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005220
Tvrtko Ursulinde1add32016-01-15 15:12:50 +00005221 file_priv->bsd_ring = -1;
5222
Ben Widawskye422b882013-12-06 14:10:58 -08005223 ret = i915_gem_context_open(dev, file);
5224 if (ret)
5225 kfree(file_priv);
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005226
Ben Widawskye422b882013-12-06 14:10:58 -08005227 return ret;
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005228}
5229
Daniel Vetterb680c372014-09-19 18:27:27 +02005230/**
5231 * i915_gem_track_fb - update frontbuffer tracking
Geliang Tangd9072a32015-09-15 05:58:44 -07005232 * @old: current GEM buffer for the frontbuffer slots
5233 * @new: new GEM buffer for the frontbuffer slots
5234 * @frontbuffer_bits: bitmask of frontbuffer slots
Daniel Vetterb680c372014-09-19 18:27:27 +02005235 *
5236 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
5237 * from @old and setting them in @new. Both @old and @new can be NULL.
5238 */
Daniel Vettera071fa02014-06-18 23:28:09 +02005239void i915_gem_track_fb(struct drm_i915_gem_object *old,
5240 struct drm_i915_gem_object *new,
5241 unsigned frontbuffer_bits)
5242{
5243 if (old) {
5244 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
5245 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
5246 old->frontbuffer_bits &= ~frontbuffer_bits;
5247 }
5248
5249 if (new) {
5250 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
5251 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
5252 new->frontbuffer_bits |= frontbuffer_bits;
5253 }
5254}
5255
Ben Widawskya70a3142013-07-31 16:59:56 -07005256/* All the new VM stuff */
Michel Thierry088e0df2015-08-07 17:40:17 +01005257u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
5258 struct i915_address_space *vm)
Ben Widawskya70a3142013-07-31 16:59:56 -07005259{
5260 struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5261 struct i915_vma *vma;
5262
Daniel Vetter896ab1a2014-08-06 15:04:51 +02005263 WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
Ben Widawskya70a3142013-07-31 16:59:56 -07005264
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00005265 list_for_each_entry(vma, &o->vma_list, obj_link) {
Chris Wilson596c5922016-02-26 11:03:20 +00005266 if (vma->is_ggtt &&
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02005267 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5268 continue;
5269 if (vma->vm == vm)
Ben Widawskya70a3142013-07-31 16:59:56 -07005270 return vma->node.start;
Ben Widawskya70a3142013-07-31 16:59:56 -07005271 }
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02005272
Daniel Vetterf25748ea2014-06-17 22:34:38 +02005273 WARN(1, "%s vma for this object not found.\n",
5274 i915_is_ggtt(vm) ? "global" : "ppgtt");
Ben Widawskya70a3142013-07-31 16:59:56 -07005275 return -1;
5276}
5277
Michel Thierry088e0df2015-08-07 17:40:17 +01005278u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
5279 const struct i915_ggtt_view *view)
Ben Widawskya70a3142013-07-31 16:59:56 -07005280{
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03005281 struct drm_i915_private *dev_priv = to_i915(o->base.dev);
5282 struct i915_ggtt *ggtt = &dev_priv->ggtt;
Ben Widawskya70a3142013-07-31 16:59:56 -07005283 struct i915_vma *vma;
5284
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00005285 list_for_each_entry(vma, &o->vma_list, obj_link)
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03005286 if (vma->vm == &ggtt->base &&
Joonas Lahtinen9abc4642015-03-27 13:09:22 +02005287 i915_ggtt_view_equal(&vma->ggtt_view, view))
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02005288 return vma->node.start;
5289
Tvrtko Ursulin5678ad72015-03-17 14:45:29 +00005290 WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02005291 return -1;
5292}
5293
5294bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
5295 struct i915_address_space *vm)
5296{
5297 struct i915_vma *vma;
5298
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00005299 list_for_each_entry(vma, &o->vma_list, obj_link) {
Chris Wilson596c5922016-02-26 11:03:20 +00005300 if (vma->is_ggtt &&
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02005301 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5302 continue;
5303 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
5304 return true;
5305 }
5306
5307 return false;
5308}
5309
5310bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
Joonas Lahtinen9abc4642015-03-27 13:09:22 +02005311 const struct i915_ggtt_view *view)
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02005312{
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03005313 struct drm_i915_private *dev_priv = to_i915(o->base.dev);
5314 struct i915_ggtt *ggtt = &dev_priv->ggtt;
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02005315 struct i915_vma *vma;
5316
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00005317 list_for_each_entry(vma, &o->vma_list, obj_link)
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03005318 if (vma->vm == &ggtt->base &&
Joonas Lahtinen9abc4642015-03-27 13:09:22 +02005319 i915_ggtt_view_equal(&vma->ggtt_view, view) &&
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00005320 drm_mm_node_allocated(&vma->node))
Ben Widawskya70a3142013-07-31 16:59:56 -07005321 return true;
5322
5323 return false;
5324}
5325
5326bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
5327{
Chris Wilson5a1d5eb2013-09-10 11:27:37 +01005328 struct i915_vma *vma;
Ben Widawskya70a3142013-07-31 16:59:56 -07005329
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00005330 list_for_each_entry(vma, &o->vma_list, obj_link)
Chris Wilson5a1d5eb2013-09-10 11:27:37 +01005331 if (drm_mm_node_allocated(&vma->node))
Ben Widawskya70a3142013-07-31 16:59:56 -07005332 return true;
5333
5334 return false;
5335}
5336
5337unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
5338 struct i915_address_space *vm)
5339{
5340 struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5341 struct i915_vma *vma;
5342
Daniel Vetter896ab1a2014-08-06 15:04:51 +02005343 WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
Ben Widawskya70a3142013-07-31 16:59:56 -07005344
5345 BUG_ON(list_empty(&o->vma_list));
5346
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00005347 list_for_each_entry(vma, &o->vma_list, obj_link) {
Chris Wilson596c5922016-02-26 11:03:20 +00005348 if (vma->is_ggtt &&
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02005349 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5350 continue;
Ben Widawskya70a3142013-07-31 16:59:56 -07005351 if (vma->vm == vm)
5352 return vma->node.size;
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02005353 }
Ben Widawskya70a3142013-07-31 16:59:56 -07005354 return 0;
5355}
5356
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02005357bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
Ben Widawsky5c2abbe2013-09-24 09:57:57 -07005358{
5359 struct i915_vma *vma;
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00005360 list_for_each_entry(vma, &obj->vma_list, obj_link)
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02005361 if (vma->pin_count > 0)
5362 return true;
Joonas Lahtinena6631ae2015-05-06 14:34:58 +03005363
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02005364 return false;
Ben Widawsky5c2abbe2013-09-24 09:57:57 -07005365}
Dave Gordonea702992015-07-09 19:29:02 +01005366
Dave Gordon033908a2015-12-10 18:51:23 +00005367/* Like i915_gem_object_get_page(), but mark the returned page dirty */
5368struct page *
5369i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
5370{
5371 struct page *page;
5372
5373 /* Only default objects have per-page dirty tracking */
Chris Wilsonde472662016-01-22 18:32:31 +00005374 if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
Dave Gordon033908a2015-12-10 18:51:23 +00005375 return NULL;
5376
5377 page = i915_gem_object_get_page(obj, n);
5378 set_page_dirty(page);
5379 return page;
5380}
5381
Dave Gordonea702992015-07-09 19:29:02 +01005382/* Allocate a new GEM object and fill it with the supplied data */
5383struct drm_i915_gem_object *
5384i915_gem_object_create_from_data(struct drm_device *dev,
5385 const void *data, size_t size)
5386{
5387 struct drm_i915_gem_object *obj;
5388 struct sg_table *sg;
5389 size_t bytes;
5390 int ret;
5391
5392 obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE));
5393 if (IS_ERR_OR_NULL(obj))
5394 return obj;
5395
5396 ret = i915_gem_object_set_to_cpu_domain(obj, true);
5397 if (ret)
5398 goto fail;
5399
5400 ret = i915_gem_object_get_pages(obj);
5401 if (ret)
5402 goto fail;
5403
5404 i915_gem_object_pin_pages(obj);
5405 sg = obj->pages;
5406 bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
Dave Gordon9e7d18c2015-12-10 18:51:24 +00005407 obj->dirty = 1; /* Backing store is now out of date */
Dave Gordonea702992015-07-09 19:29:02 +01005408 i915_gem_object_unpin_pages(obj);
5409
5410 if (WARN_ON(bytes != size)) {
5411 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
5412 ret = -EFAULT;
5413 goto fail;
5414 }
5415
5416 return obj;
5417
5418fail:
5419 drm_gem_object_unreference(&obj->base);
5420 return ERR_PTR(ret);
5421}