blob: 977a0aa5ecd01fe0f08c15a2c25c8873d856d896 [file] [log] [blame]
Eric Anholt673a3942008-07-30 12:06:12 -07001/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
David Howells760285e2012-10-02 18:01:07 +010028#include <drm/drmP.h>
29#include <drm/i915_drm.h>
Eric Anholt673a3942008-07-30 12:06:12 -070030#include "i915_drv.h"
Chris Wilson1c5d22f2009-08-25 11:15:50 +010031#include "i915_trace.h"
Jesse Barnes652c3932009-08-17 13:31:43 -070032#include "intel_drv.h"
Hugh Dickins5949eac2011-06-27 16:18:18 -070033#include <linux/shmem_fs.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090034#include <linux/slab.h>
Eric Anholt673a3942008-07-30 12:06:12 -070035#include <linux/swap.h>
Jesse Barnes79e53942008-11-07 14:24:08 -080036#include <linux/pci.h>
Daniel Vetter1286ff72012-05-10 15:25:09 +020037#include <linux/dma-buf.h>
Eric Anholt673a3942008-07-30 12:06:12 -070038
Chris Wilson05394f32010-11-08 19:18:58 +000039static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
40static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
Chris Wilson88241782011-01-07 17:09:48 +000041static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
42 unsigned alignment,
Chris Wilson86a1ee22012-08-11 15:41:04 +010043 bool map_and_fenceable,
44 bool nonblocking);
Chris Wilson05394f32010-11-08 19:18:58 +000045static int i915_gem_phys_pwrite(struct drm_device *dev,
46 struct drm_i915_gem_object *obj,
Dave Airlie71acb5e2008-12-30 20:31:46 +100047 struct drm_i915_gem_pwrite *args,
Chris Wilson05394f32010-11-08 19:18:58 +000048 struct drm_file *file);
Eric Anholt673a3942008-07-30 12:06:12 -070049
Chris Wilson61050802012-04-17 15:31:31 +010050static void i915_gem_write_fence(struct drm_device *dev, int reg,
51 struct drm_i915_gem_object *obj);
52static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
53 struct drm_i915_fence_reg *fence,
54 bool enable);
55
Chris Wilson17250b72010-10-28 12:51:39 +010056static int i915_gem_inactive_shrink(struct shrinker *shrinker,
Ying Han1495f232011-05-24 17:12:27 -070057 struct shrink_control *sc);
Chris Wilson6c085a72012-08-20 11:40:46 +020058static long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
59static void i915_gem_shrink_all(struct drm_i915_private *dev_priv);
Daniel Vetter8c599672011-12-14 13:57:31 +010060static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
Chris Wilson31169712009-09-14 16:50:28 +010061
Chris Wilson61050802012-04-17 15:31:31 +010062static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
63{
64 if (obj->tiling_mode)
65 i915_gem_release_mmap(obj);
66
67 /* As we do not have an associated fence register, we will force
68 * a tiling change if we ever need to acquire one.
69 */
Chris Wilson5d82e3e2012-04-21 16:23:23 +010070 obj->fence_dirty = false;
Chris Wilson61050802012-04-17 15:31:31 +010071 obj->fence_reg = I915_FENCE_REG_NONE;
72}
73
Chris Wilson73aa8082010-09-30 11:46:12 +010074/* some bookkeeping */
75static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
76 size_t size)
77{
78 dev_priv->mm.object_count++;
79 dev_priv->mm.object_memory += size;
80}
81
82static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
83 size_t size)
84{
85 dev_priv->mm.object_count--;
86 dev_priv->mm.object_memory -= size;
87}
88
Chris Wilson21dd3732011-01-26 15:55:56 +000089static int
Daniel Vetter33196de2012-11-14 17:14:05 +010090i915_gem_wait_for_error(struct i915_gpu_error *error)
Chris Wilson30dbf0c2010-09-25 10:19:17 +010091{
Chris Wilson30dbf0c2010-09-25 10:19:17 +010092 int ret;
93
Daniel Vetter1f83fee2012-11-15 17:17:22 +010094#define EXIT_COND (!i915_reset_in_progress(error))
95 if (EXIT_COND)
Chris Wilson30dbf0c2010-09-25 10:19:17 +010096 return 0;
97
Daniel Vetter1f83fee2012-11-15 17:17:22 +010098 /* GPU is already declared terminally dead, give up. */
99 if (i915_terminally_wedged(error))
100 return -EIO;
101
Daniel Vetter0a6759c2012-07-04 22:18:41 +0200102 /*
103 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
104 * userspace. If it takes that long something really bad is going on and
105 * we should simply try to bail out and fail as gracefully as possible.
106 */
Daniel Vetter1f83fee2012-11-15 17:17:22 +0100107 ret = wait_event_interruptible_timeout(error->reset_queue,
108 EXIT_COND,
109 10*HZ);
Daniel Vetter0a6759c2012-07-04 22:18:41 +0200110 if (ret == 0) {
111 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
112 return -EIO;
113 } else if (ret < 0) {
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100114 return ret;
Daniel Vetter0a6759c2012-07-04 22:18:41 +0200115 }
Daniel Vetter1f83fee2012-11-15 17:17:22 +0100116#undef EXIT_COND
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100117
Chris Wilson21dd3732011-01-26 15:55:56 +0000118 return 0;
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100119}
120
Chris Wilson54cf91d2010-11-25 18:00:26 +0000121int i915_mutex_lock_interruptible(struct drm_device *dev)
Chris Wilson76c1dec2010-09-25 11:22:51 +0100122{
Daniel Vetter33196de2012-11-14 17:14:05 +0100123 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson76c1dec2010-09-25 11:22:51 +0100124 int ret;
125
Daniel Vetter33196de2012-11-14 17:14:05 +0100126 ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
Chris Wilson76c1dec2010-09-25 11:22:51 +0100127 if (ret)
128 return ret;
129
130 ret = mutex_lock_interruptible(&dev->struct_mutex);
131 if (ret)
132 return ret;
133
Chris Wilson23bc5982010-09-29 16:10:57 +0100134 WARN_ON(i915_verify_lists(dev));
Chris Wilson76c1dec2010-09-25 11:22:51 +0100135 return 0;
136}
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100137
Chris Wilson7d1c4802010-08-07 21:45:03 +0100138static inline bool
Chris Wilson05394f32010-11-08 19:18:58 +0000139i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
Chris Wilson7d1c4802010-08-07 21:45:03 +0100140{
Chris Wilson6c085a72012-08-20 11:40:46 +0200141 return obj->gtt_space && !obj->active;
Chris Wilson7d1c4802010-08-07 21:45:03 +0100142}
143
Eric Anholt673a3942008-07-30 12:06:12 -0700144int
145i915_gem_init_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +0000146 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -0700147{
Ben Widawsky93d18792013-01-17 12:45:17 -0800148 struct drm_i915_private *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -0700149 struct drm_i915_gem_init *args = data;
Chris Wilson20217462010-11-23 15:26:33 +0000150
Daniel Vetter7bb6fb82012-04-24 08:22:52 +0200151 if (drm_core_check_feature(dev, DRIVER_MODESET))
152 return -ENODEV;
153
Chris Wilson20217462010-11-23 15:26:33 +0000154 if (args->gtt_start >= args->gtt_end ||
155 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
156 return -EINVAL;
Eric Anholt673a3942008-07-30 12:06:12 -0700157
Daniel Vetterf534bc02012-03-26 22:37:04 +0200158 /* GEM with user mode setting was never supported on ilk and later. */
159 if (INTEL_INFO(dev)->gen >= 5)
160 return -ENODEV;
161
Eric Anholt673a3942008-07-30 12:06:12 -0700162 mutex_lock(&dev->struct_mutex);
Ben Widawskyd7e50082012-12-18 10:31:25 -0800163 i915_gem_setup_global_gtt(dev, args->gtt_start, args->gtt_end,
164 args->gtt_end);
Ben Widawsky93d18792013-01-17 12:45:17 -0800165 dev_priv->gtt.mappable_end = args->gtt_end;
Eric Anholt673a3942008-07-30 12:06:12 -0700166 mutex_unlock(&dev->struct_mutex);
167
Chris Wilson20217462010-11-23 15:26:33 +0000168 return 0;
Eric Anholt673a3942008-07-30 12:06:12 -0700169}
170
Eric Anholt5a125c32008-10-22 21:40:13 -0700171int
172i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +0000173 struct drm_file *file)
Eric Anholt5a125c32008-10-22 21:40:13 -0700174{
Chris Wilson73aa8082010-09-30 11:46:12 +0100175 struct drm_i915_private *dev_priv = dev->dev_private;
Eric Anholt5a125c32008-10-22 21:40:13 -0700176 struct drm_i915_gem_get_aperture *args = data;
Chris Wilson6299f992010-11-24 12:23:44 +0000177 struct drm_i915_gem_object *obj;
178 size_t pinned;
Eric Anholt5a125c32008-10-22 21:40:13 -0700179
Chris Wilson6299f992010-11-24 12:23:44 +0000180 pinned = 0;
Chris Wilson73aa8082010-09-30 11:46:12 +0100181 mutex_lock(&dev->struct_mutex);
Chris Wilson6c085a72012-08-20 11:40:46 +0200182 list_for_each_entry(obj, &dev_priv->mm.bound_list, gtt_list)
Chris Wilson1b502472012-04-24 15:47:30 +0100183 if (obj->pin_count)
184 pinned += obj->gtt_space->size;
Chris Wilson73aa8082010-09-30 11:46:12 +0100185 mutex_unlock(&dev->struct_mutex);
Eric Anholt5a125c32008-10-22 21:40:13 -0700186
Ben Widawsky5d4545a2013-01-17 12:45:15 -0800187 args->aper_size = dev_priv->gtt.total;
Akshay Joshi0206e352011-08-16 15:34:10 -0400188 args->aper_available_size = args->aper_size - pinned;
Chris Wilson6299f992010-11-24 12:23:44 +0000189
Eric Anholt5a125c32008-10-22 21:40:13 -0700190 return 0;
191}
192
Chris Wilson42dcedd2012-11-15 11:32:30 +0000193void *i915_gem_object_alloc(struct drm_device *dev)
194{
195 struct drm_i915_private *dev_priv = dev->dev_private;
196 return kmem_cache_alloc(dev_priv->slab, GFP_KERNEL | __GFP_ZERO);
197}
198
199void i915_gem_object_free(struct drm_i915_gem_object *obj)
200{
201 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
202 kmem_cache_free(dev_priv->slab, obj);
203}
204
Dave Airlieff72145b2011-02-07 12:16:14 +1000205static int
206i915_gem_create(struct drm_file *file,
207 struct drm_device *dev,
208 uint64_t size,
209 uint32_t *handle_p)
Eric Anholt673a3942008-07-30 12:06:12 -0700210{
Chris Wilson05394f32010-11-08 19:18:58 +0000211 struct drm_i915_gem_object *obj;
Pekka Paalanena1a2d1d2009-08-23 12:40:55 +0300212 int ret;
213 u32 handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700214
Dave Airlieff72145b2011-02-07 12:16:14 +1000215 size = roundup(size, PAGE_SIZE);
Chris Wilson8ffc0242011-09-14 14:14:28 +0200216 if (size == 0)
217 return -EINVAL;
Eric Anholt673a3942008-07-30 12:06:12 -0700218
219 /* Allocate the new object */
Dave Airlieff72145b2011-02-07 12:16:14 +1000220 obj = i915_gem_alloc_object(dev, size);
Eric Anholt673a3942008-07-30 12:06:12 -0700221 if (obj == NULL)
222 return -ENOMEM;
223
Chris Wilson05394f32010-11-08 19:18:58 +0000224 ret = drm_gem_handle_create(file, &obj->base, &handle);
Chris Wilson1dfd9752010-09-06 14:44:14 +0100225 if (ret) {
Chris Wilson05394f32010-11-08 19:18:58 +0000226 drm_gem_object_release(&obj->base);
227 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
Chris Wilson42dcedd2012-11-15 11:32:30 +0000228 i915_gem_object_free(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700229 return ret;
Chris Wilson1dfd9752010-09-06 14:44:14 +0100230 }
231
Chris Wilson202f2fe2010-10-14 13:20:40 +0100232 /* drop reference from allocate - handle holds it now */
Chris Wilson05394f32010-11-08 19:18:58 +0000233 drm_gem_object_unreference(&obj->base);
Chris Wilson202f2fe2010-10-14 13:20:40 +0100234 trace_i915_gem_object_create(obj);
235
Dave Airlieff72145b2011-02-07 12:16:14 +1000236 *handle_p = handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700237 return 0;
238}
239
Dave Airlieff72145b2011-02-07 12:16:14 +1000240int
241i915_gem_dumb_create(struct drm_file *file,
242 struct drm_device *dev,
243 struct drm_mode_create_dumb *args)
244{
245 /* have to work out size/pitch and return them */
Chris Wilsoned0291f2011-03-19 08:21:45 +0000246 args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
Dave Airlieff72145b2011-02-07 12:16:14 +1000247 args->size = args->pitch * args->height;
248 return i915_gem_create(file, dev,
249 args->size, &args->handle);
250}
251
252int i915_gem_dumb_destroy(struct drm_file *file,
253 struct drm_device *dev,
254 uint32_t handle)
255{
256 return drm_gem_handle_delete(file, handle);
257}
258
259/**
260 * Creates a new mm object and returns a handle to it.
261 */
262int
263i915_gem_create_ioctl(struct drm_device *dev, void *data,
264 struct drm_file *file)
265{
266 struct drm_i915_gem_create *args = data;
Daniel Vetter63ed2cb2012-04-23 16:50:50 +0200267
Dave Airlieff72145b2011-02-07 12:16:14 +1000268 return i915_gem_create(file, dev,
269 args->size, &args->handle);
270}
271
Daniel Vetter8c599672011-12-14 13:57:31 +0100272static inline int
Daniel Vetter8461d222011-12-14 13:57:32 +0100273__copy_to_user_swizzled(char __user *cpu_vaddr,
274 const char *gpu_vaddr, int gpu_offset,
275 int length)
276{
277 int ret, cpu_offset = 0;
278
279 while (length > 0) {
280 int cacheline_end = ALIGN(gpu_offset + 1, 64);
281 int this_length = min(cacheline_end - gpu_offset, length);
282 int swizzled_gpu_offset = gpu_offset ^ 64;
283
284 ret = __copy_to_user(cpu_vaddr + cpu_offset,
285 gpu_vaddr + swizzled_gpu_offset,
286 this_length);
287 if (ret)
288 return ret + length;
289
290 cpu_offset += this_length;
291 gpu_offset += this_length;
292 length -= this_length;
293 }
294
295 return 0;
296}
297
298static inline int
Ben Widawsky4f0c7cf2012-04-16 14:07:47 -0700299__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
300 const char __user *cpu_vaddr,
Daniel Vetter8c599672011-12-14 13:57:31 +0100301 int length)
302{
303 int ret, cpu_offset = 0;
304
305 while (length > 0) {
306 int cacheline_end = ALIGN(gpu_offset + 1, 64);
307 int this_length = min(cacheline_end - gpu_offset, length);
308 int swizzled_gpu_offset = gpu_offset ^ 64;
309
310 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
311 cpu_vaddr + cpu_offset,
312 this_length);
313 if (ret)
314 return ret + length;
315
316 cpu_offset += this_length;
317 gpu_offset += this_length;
318 length -= this_length;
319 }
320
321 return 0;
322}
323
Daniel Vetterd174bd62012-03-25 19:47:40 +0200324/* Per-page copy function for the shmem pread fastpath.
325 * Flushes invalid cachelines before reading the target if
326 * needs_clflush is set. */
Eric Anholteb014592009-03-10 11:44:52 -0700327static int
Daniel Vetterd174bd62012-03-25 19:47:40 +0200328shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
329 char __user *user_data,
330 bool page_do_bit17_swizzling, bool needs_clflush)
331{
332 char *vaddr;
333 int ret;
334
Daniel Vettere7e58eb2012-03-25 19:47:43 +0200335 if (unlikely(page_do_bit17_swizzling))
Daniel Vetterd174bd62012-03-25 19:47:40 +0200336 return -EINVAL;
337
338 vaddr = kmap_atomic(page);
339 if (needs_clflush)
340 drm_clflush_virt_range(vaddr + shmem_page_offset,
341 page_length);
342 ret = __copy_to_user_inatomic(user_data,
343 vaddr + shmem_page_offset,
344 page_length);
345 kunmap_atomic(vaddr);
346
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100347 return ret ? -EFAULT : 0;
Daniel Vetterd174bd62012-03-25 19:47:40 +0200348}
349
Daniel Vetter23c18c72012-03-25 19:47:42 +0200350static void
351shmem_clflush_swizzled_range(char *addr, unsigned long length,
352 bool swizzled)
353{
Daniel Vettere7e58eb2012-03-25 19:47:43 +0200354 if (unlikely(swizzled)) {
Daniel Vetter23c18c72012-03-25 19:47:42 +0200355 unsigned long start = (unsigned long) addr;
356 unsigned long end = (unsigned long) addr + length;
357
358 /* For swizzling simply ensure that we always flush both
359 * channels. Lame, but simple and it works. Swizzled
360 * pwrite/pread is far from a hotpath - current userspace
361 * doesn't use it at all. */
362 start = round_down(start, 128);
363 end = round_up(end, 128);
364
365 drm_clflush_virt_range((void *)start, end - start);
366 } else {
367 drm_clflush_virt_range(addr, length);
368 }
369
370}
371
Daniel Vetterd174bd62012-03-25 19:47:40 +0200372/* Only difference to the fast-path function is that this can handle bit17
373 * and uses non-atomic copy and kmap functions. */
374static int
375shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
376 char __user *user_data,
377 bool page_do_bit17_swizzling, bool needs_clflush)
378{
379 char *vaddr;
380 int ret;
381
382 vaddr = kmap(page);
383 if (needs_clflush)
Daniel Vetter23c18c72012-03-25 19:47:42 +0200384 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
385 page_length,
386 page_do_bit17_swizzling);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200387
388 if (page_do_bit17_swizzling)
389 ret = __copy_to_user_swizzled(user_data,
390 vaddr, shmem_page_offset,
391 page_length);
392 else
393 ret = __copy_to_user(user_data,
394 vaddr + shmem_page_offset,
395 page_length);
396 kunmap(page);
397
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100398 return ret ? - EFAULT : 0;
Daniel Vetterd174bd62012-03-25 19:47:40 +0200399}
400
Eric Anholteb014592009-03-10 11:44:52 -0700401static int
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200402i915_gem_shmem_pread(struct drm_device *dev,
403 struct drm_i915_gem_object *obj,
404 struct drm_i915_gem_pread *args,
405 struct drm_file *file)
Eric Anholteb014592009-03-10 11:44:52 -0700406{
Daniel Vetter8461d222011-12-14 13:57:32 +0100407 char __user *user_data;
Eric Anholteb014592009-03-10 11:44:52 -0700408 ssize_t remain;
Daniel Vetter8461d222011-12-14 13:57:32 +0100409 loff_t offset;
Ben Widawskyeb2c0c82012-02-15 14:42:43 +0100410 int shmem_page_offset, page_length, ret = 0;
Daniel Vetter8461d222011-12-14 13:57:32 +0100411 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
Daniel Vetter96d79b52012-03-25 19:47:36 +0200412 int prefaulted = 0;
Daniel Vetter84897312012-03-25 19:47:31 +0200413 int needs_clflush = 0;
Imre Deak67d5a502013-02-18 19:28:02 +0200414 struct sg_page_iter sg_iter;
Eric Anholteb014592009-03-10 11:44:52 -0700415
Ville Syrjälä2bb46292013-02-22 16:12:51 +0200416 user_data = to_user_ptr(args->data_ptr);
Eric Anholteb014592009-03-10 11:44:52 -0700417 remain = args->size;
418
Daniel Vetter8461d222011-12-14 13:57:32 +0100419 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700420
Daniel Vetter84897312012-03-25 19:47:31 +0200421 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
422 /* If we're not in the cpu read domain, set ourself into the gtt
423 * read domain and manually flush cachelines (if required). This
424 * optimizes for the case when the gpu will dirty the data
425 * anyway again before the next pread happens. */
426 if (obj->cache_level == I915_CACHE_NONE)
427 needs_clflush = 1;
Chris Wilson6c085a72012-08-20 11:40:46 +0200428 if (obj->gtt_space) {
429 ret = i915_gem_object_set_to_gtt_domain(obj, false);
430 if (ret)
431 return ret;
432 }
Daniel Vetter84897312012-03-25 19:47:31 +0200433 }
Eric Anholteb014592009-03-10 11:44:52 -0700434
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100435 ret = i915_gem_object_get_pages(obj);
436 if (ret)
437 return ret;
438
439 i915_gem_object_pin_pages(obj);
440
Eric Anholteb014592009-03-10 11:44:52 -0700441 offset = args->offset;
Daniel Vetter8461d222011-12-14 13:57:32 +0100442
Imre Deak67d5a502013-02-18 19:28:02 +0200443 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
444 offset >> PAGE_SHIFT) {
Imre Deak2db76d72013-03-26 15:14:18 +0200445 struct page *page = sg_page_iter_page(&sg_iter);
Chris Wilson9da3da62012-06-01 15:20:22 +0100446
447 if (remain <= 0)
448 break;
449
Eric Anholteb014592009-03-10 11:44:52 -0700450 /* Operation in this page
451 *
Eric Anholteb014592009-03-10 11:44:52 -0700452 * shmem_page_offset = offset within page in shmem file
Eric Anholteb014592009-03-10 11:44:52 -0700453 * page_length = bytes to copy for this page
454 */
Chris Wilsonc8cbbb82011-05-12 22:17:11 +0100455 shmem_page_offset = offset_in_page(offset);
Eric Anholteb014592009-03-10 11:44:52 -0700456 page_length = remain;
457 if ((shmem_page_offset + page_length) > PAGE_SIZE)
458 page_length = PAGE_SIZE - shmem_page_offset;
Eric Anholteb014592009-03-10 11:44:52 -0700459
Daniel Vetter8461d222011-12-14 13:57:32 +0100460 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
461 (page_to_phys(page) & (1 << 17)) != 0;
462
Daniel Vetterd174bd62012-03-25 19:47:40 +0200463 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
464 user_data, page_do_bit17_swizzling,
465 needs_clflush);
466 if (ret == 0)
467 goto next_page;
Eric Anholteb014592009-03-10 11:44:52 -0700468
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200469 mutex_unlock(&dev->struct_mutex);
470
Daniel Vetter96d79b52012-03-25 19:47:36 +0200471 if (!prefaulted) {
Daniel Vetterf56f8212012-03-25 19:47:41 +0200472 ret = fault_in_multipages_writeable(user_data, remain);
Daniel Vetter96d79b52012-03-25 19:47:36 +0200473 /* Userspace is tricking us, but we've already clobbered
474 * its pages with the prefault and promised to write the
475 * data up to the first fault. Hence ignore any errors
476 * and just continue. */
477 (void)ret;
478 prefaulted = 1;
479 }
480
Daniel Vetterd174bd62012-03-25 19:47:40 +0200481 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
482 user_data, page_do_bit17_swizzling,
483 needs_clflush);
Eric Anholteb014592009-03-10 11:44:52 -0700484
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200485 mutex_lock(&dev->struct_mutex);
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100486
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200487next_page:
Chris Wilsone5281cc2010-10-28 13:45:36 +0100488 mark_page_accessed(page);
Chris Wilsone5281cc2010-10-28 13:45:36 +0100489
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100490 if (ret)
Daniel Vetter8461d222011-12-14 13:57:32 +0100491 goto out;
Daniel Vetter8461d222011-12-14 13:57:32 +0100492
Eric Anholteb014592009-03-10 11:44:52 -0700493 remain -= page_length;
Daniel Vetter8461d222011-12-14 13:57:32 +0100494 user_data += page_length;
Eric Anholteb014592009-03-10 11:44:52 -0700495 offset += page_length;
496 }
497
Chris Wilson4f27b752010-10-14 15:26:45 +0100498out:
Chris Wilsonf60d7f02012-09-04 21:02:56 +0100499 i915_gem_object_unpin_pages(obj);
500
Eric Anholteb014592009-03-10 11:44:52 -0700501 return ret;
502}
503
Eric Anholt673a3942008-07-30 12:06:12 -0700504/**
505 * Reads data from the object referenced by handle.
506 *
507 * On error, the contents of *data are undefined.
508 */
509int
510i915_gem_pread_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +0000511 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -0700512{
513 struct drm_i915_gem_pread *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +0000514 struct drm_i915_gem_object *obj;
Chris Wilson35b62a82010-09-26 20:23:38 +0100515 int ret = 0;
Eric Anholt673a3942008-07-30 12:06:12 -0700516
Chris Wilson51311d02010-11-17 09:10:42 +0000517 if (args->size == 0)
518 return 0;
519
520 if (!access_ok(VERIFY_WRITE,
Ville Syrjälä2bb46292013-02-22 16:12:51 +0200521 to_user_ptr(args->data_ptr),
Chris Wilson51311d02010-11-17 09:10:42 +0000522 args->size))
523 return -EFAULT;
524
Chris Wilson4f27b752010-10-14 15:26:45 +0100525 ret = i915_mutex_lock_interruptible(dev);
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100526 if (ret)
Chris Wilson4f27b752010-10-14 15:26:45 +0100527 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700528
Chris Wilson05394f32010-11-08 19:18:58 +0000529 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +0000530 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100531 ret = -ENOENT;
532 goto unlock;
Chris Wilson4f27b752010-10-14 15:26:45 +0100533 }
Eric Anholt673a3942008-07-30 12:06:12 -0700534
Chris Wilson7dcd2492010-09-26 20:21:44 +0100535 /* Bounds check source. */
Chris Wilson05394f32010-11-08 19:18:58 +0000536 if (args->offset > obj->base.size ||
537 args->size > obj->base.size - args->offset) {
Chris Wilsonce9d4192010-09-26 20:50:05 +0100538 ret = -EINVAL;
Chris Wilson35b62a82010-09-26 20:23:38 +0100539 goto out;
Chris Wilsonce9d4192010-09-26 20:50:05 +0100540 }
541
Daniel Vetter1286ff72012-05-10 15:25:09 +0200542 /* prime objects have no backing filp to GEM pread/pwrite
543 * pages from.
544 */
545 if (!obj->base.filp) {
546 ret = -EINVAL;
547 goto out;
548 }
549
Chris Wilsondb53a302011-02-03 11:57:46 +0000550 trace_i915_gem_object_pread(obj, args->offset, args->size);
551
Daniel Vetterdbf7bff2012-03-25 19:47:29 +0200552 ret = i915_gem_shmem_pread(dev, obj, args, file);
Eric Anholt673a3942008-07-30 12:06:12 -0700553
Chris Wilson35b62a82010-09-26 20:23:38 +0100554out:
Chris Wilson05394f32010-11-08 19:18:58 +0000555 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100556unlock:
Chris Wilson4f27b752010-10-14 15:26:45 +0100557 mutex_unlock(&dev->struct_mutex);
Eric Anholteb014592009-03-10 11:44:52 -0700558 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700559}
560
Keith Packard0839ccb2008-10-30 19:38:48 -0700561/* This is the fast write path which cannot handle
562 * page faults in the source data
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700563 */
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700564
Keith Packard0839ccb2008-10-30 19:38:48 -0700565static inline int
566fast_user_write(struct io_mapping *mapping,
567 loff_t page_base, int page_offset,
568 char __user *user_data,
569 int length)
570{
Ben Widawsky4f0c7cf2012-04-16 14:07:47 -0700571 void __iomem *vaddr_atomic;
572 void *vaddr;
Keith Packard0839ccb2008-10-30 19:38:48 -0700573 unsigned long unwritten;
574
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -0700575 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
Ben Widawsky4f0c7cf2012-04-16 14:07:47 -0700576 /* We can use the cpu mem copy function because this is X86. */
577 vaddr = (void __force*)vaddr_atomic + page_offset;
578 unwritten = __copy_from_user_inatomic_nocache(vaddr,
Keith Packard0839ccb2008-10-30 19:38:48 -0700579 user_data, length);
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -0700580 io_mapping_unmap_atomic(vaddr_atomic);
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100581 return unwritten;
Keith Packard0839ccb2008-10-30 19:38:48 -0700582}
583
Eric Anholt3de09aa2009-03-09 09:42:23 -0700584/**
585 * This is the fast pwrite path, where we copy the data directly from the
586 * user into the GTT, uncached.
587 */
Eric Anholt673a3942008-07-30 12:06:12 -0700588static int
Chris Wilson05394f32010-11-08 19:18:58 +0000589i915_gem_gtt_pwrite_fast(struct drm_device *dev,
590 struct drm_i915_gem_object *obj,
Eric Anholt3de09aa2009-03-09 09:42:23 -0700591 struct drm_i915_gem_pwrite *args,
Chris Wilson05394f32010-11-08 19:18:58 +0000592 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -0700593{
Keith Packard0839ccb2008-10-30 19:38:48 -0700594 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -0700595 ssize_t remain;
Keith Packard0839ccb2008-10-30 19:38:48 -0700596 loff_t offset, page_base;
Eric Anholt673a3942008-07-30 12:06:12 -0700597 char __user *user_data;
Daniel Vetter935aaa62012-03-25 19:47:35 +0200598 int page_offset, page_length, ret;
599
Chris Wilson86a1ee22012-08-11 15:41:04 +0100600 ret = i915_gem_object_pin(obj, 0, true, true);
Daniel Vetter935aaa62012-03-25 19:47:35 +0200601 if (ret)
602 goto out;
603
604 ret = i915_gem_object_set_to_gtt_domain(obj, true);
605 if (ret)
606 goto out_unpin;
607
608 ret = i915_gem_object_put_fence(obj);
609 if (ret)
610 goto out_unpin;
Eric Anholt673a3942008-07-30 12:06:12 -0700611
Ville Syrjälä2bb46292013-02-22 16:12:51 +0200612 user_data = to_user_ptr(args->data_ptr);
Eric Anholt673a3942008-07-30 12:06:12 -0700613 remain = args->size;
Eric Anholt673a3942008-07-30 12:06:12 -0700614
Chris Wilson05394f32010-11-08 19:18:58 +0000615 offset = obj->gtt_offset + args->offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700616
617 while (remain > 0) {
618 /* Operation in this page
619 *
Keith Packard0839ccb2008-10-30 19:38:48 -0700620 * page_base = page offset within aperture
621 * page_offset = offset within page
622 * page_length = bytes to copy for this page
Eric Anholt673a3942008-07-30 12:06:12 -0700623 */
Chris Wilsonc8cbbb82011-05-12 22:17:11 +0100624 page_base = offset & PAGE_MASK;
625 page_offset = offset_in_page(offset);
Keith Packard0839ccb2008-10-30 19:38:48 -0700626 page_length = remain;
627 if ((page_offset + remain) > PAGE_SIZE)
628 page_length = PAGE_SIZE - page_offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700629
Keith Packard0839ccb2008-10-30 19:38:48 -0700630 /* If we get a fault while copying data, then (presumably) our
Eric Anholt3de09aa2009-03-09 09:42:23 -0700631 * source page isn't available. Return the error and we'll
632 * retry in the slow path.
Keith Packard0839ccb2008-10-30 19:38:48 -0700633 */
Ben Widawsky5d4545a2013-01-17 12:45:15 -0800634 if (fast_user_write(dev_priv->gtt.mappable, page_base,
Daniel Vetter935aaa62012-03-25 19:47:35 +0200635 page_offset, user_data, page_length)) {
636 ret = -EFAULT;
637 goto out_unpin;
638 }
Eric Anholt673a3942008-07-30 12:06:12 -0700639
Keith Packard0839ccb2008-10-30 19:38:48 -0700640 remain -= page_length;
641 user_data += page_length;
642 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700643 }
Eric Anholt673a3942008-07-30 12:06:12 -0700644
Daniel Vetter935aaa62012-03-25 19:47:35 +0200645out_unpin:
646 i915_gem_object_unpin(obj);
647out:
Eric Anholt3de09aa2009-03-09 09:42:23 -0700648 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700649}
650
Daniel Vetterd174bd62012-03-25 19:47:40 +0200651/* Per-page copy function for the shmem pwrite fastpath.
652 * Flushes invalid cachelines before writing to the target if
653 * needs_clflush_before is set and flushes out any written cachelines after
654 * writing if needs_clflush is set. */
Eric Anholt673a3942008-07-30 12:06:12 -0700655static int
Daniel Vetterd174bd62012-03-25 19:47:40 +0200656shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
657 char __user *user_data,
658 bool page_do_bit17_swizzling,
659 bool needs_clflush_before,
660 bool needs_clflush_after)
Eric Anholt673a3942008-07-30 12:06:12 -0700661{
Daniel Vetterd174bd62012-03-25 19:47:40 +0200662 char *vaddr;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700663 int ret;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700664
Daniel Vettere7e58eb2012-03-25 19:47:43 +0200665 if (unlikely(page_do_bit17_swizzling))
Daniel Vetterd174bd62012-03-25 19:47:40 +0200666 return -EINVAL;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700667
Daniel Vetterd174bd62012-03-25 19:47:40 +0200668 vaddr = kmap_atomic(page);
669 if (needs_clflush_before)
670 drm_clflush_virt_range(vaddr + shmem_page_offset,
671 page_length);
672 ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
673 user_data,
674 page_length);
675 if (needs_clflush_after)
676 drm_clflush_virt_range(vaddr + shmem_page_offset,
677 page_length);
678 kunmap_atomic(vaddr);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700679
Chris Wilson755d2212012-09-04 21:02:55 +0100680 return ret ? -EFAULT : 0;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700681}
682
Daniel Vetterd174bd62012-03-25 19:47:40 +0200683/* Only difference to the fast-path function is that this can handle bit17
684 * and uses non-atomic copy and kmap functions. */
Eric Anholt3043c602008-10-02 12:24:47 -0700685static int
Daniel Vetterd174bd62012-03-25 19:47:40 +0200686shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
687 char __user *user_data,
688 bool page_do_bit17_swizzling,
689 bool needs_clflush_before,
690 bool needs_clflush_after)
Eric Anholt673a3942008-07-30 12:06:12 -0700691{
Daniel Vetterd174bd62012-03-25 19:47:40 +0200692 char *vaddr;
693 int ret;
Eric Anholt40123c12009-03-09 13:42:30 -0700694
Daniel Vetterd174bd62012-03-25 19:47:40 +0200695 vaddr = kmap(page);
Daniel Vettere7e58eb2012-03-25 19:47:43 +0200696 if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
Daniel Vetter23c18c72012-03-25 19:47:42 +0200697 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
698 page_length,
699 page_do_bit17_swizzling);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200700 if (page_do_bit17_swizzling)
701 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
Chris Wilsone5281cc2010-10-28 13:45:36 +0100702 user_data,
703 page_length);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200704 else
705 ret = __copy_from_user(vaddr + shmem_page_offset,
706 user_data,
707 page_length);
708 if (needs_clflush_after)
Daniel Vetter23c18c72012-03-25 19:47:42 +0200709 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
710 page_length,
711 page_do_bit17_swizzling);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200712 kunmap(page);
Chris Wilsone5281cc2010-10-28 13:45:36 +0100713
Chris Wilson755d2212012-09-04 21:02:55 +0100714 return ret ? -EFAULT : 0;
Eric Anholt40123c12009-03-09 13:42:30 -0700715}
716
Eric Anholt40123c12009-03-09 13:42:30 -0700717static int
Daniel Vettere244a442012-03-25 19:47:28 +0200718i915_gem_shmem_pwrite(struct drm_device *dev,
719 struct drm_i915_gem_object *obj,
720 struct drm_i915_gem_pwrite *args,
721 struct drm_file *file)
Eric Anholt40123c12009-03-09 13:42:30 -0700722{
Eric Anholt40123c12009-03-09 13:42:30 -0700723 ssize_t remain;
Daniel Vetter8c599672011-12-14 13:57:31 +0100724 loff_t offset;
725 char __user *user_data;
Ben Widawskyeb2c0c82012-02-15 14:42:43 +0100726 int shmem_page_offset, page_length, ret = 0;
Daniel Vetter8c599672011-12-14 13:57:31 +0100727 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
Daniel Vettere244a442012-03-25 19:47:28 +0200728 int hit_slowpath = 0;
Daniel Vetter58642882012-03-25 19:47:37 +0200729 int needs_clflush_after = 0;
730 int needs_clflush_before = 0;
Imre Deak67d5a502013-02-18 19:28:02 +0200731 struct sg_page_iter sg_iter;
Eric Anholt40123c12009-03-09 13:42:30 -0700732
Ville Syrjälä2bb46292013-02-22 16:12:51 +0200733 user_data = to_user_ptr(args->data_ptr);
Eric Anholt40123c12009-03-09 13:42:30 -0700734 remain = args->size;
735
Daniel Vetter8c599672011-12-14 13:57:31 +0100736 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
Eric Anholt40123c12009-03-09 13:42:30 -0700737
Daniel Vetter58642882012-03-25 19:47:37 +0200738 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
739 /* If we're not in the cpu write domain, set ourself into the gtt
740 * write domain and manually flush cachelines (if required). This
741 * optimizes for the case when the gpu will use the data
742 * right away and we therefore have to clflush anyway. */
743 if (obj->cache_level == I915_CACHE_NONE)
744 needs_clflush_after = 1;
Chris Wilson6c085a72012-08-20 11:40:46 +0200745 if (obj->gtt_space) {
746 ret = i915_gem_object_set_to_gtt_domain(obj, true);
747 if (ret)
748 return ret;
749 }
Daniel Vetter58642882012-03-25 19:47:37 +0200750 }
751 /* Same trick applies for invalidate partially written cachelines before
752 * writing. */
753 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)
754 && obj->cache_level == I915_CACHE_NONE)
755 needs_clflush_before = 1;
756
Chris Wilson755d2212012-09-04 21:02:55 +0100757 ret = i915_gem_object_get_pages(obj);
758 if (ret)
759 return ret;
760
761 i915_gem_object_pin_pages(obj);
762
Eric Anholt40123c12009-03-09 13:42:30 -0700763 offset = args->offset;
Chris Wilson05394f32010-11-08 19:18:58 +0000764 obj->dirty = 1;
Eric Anholt40123c12009-03-09 13:42:30 -0700765
Imre Deak67d5a502013-02-18 19:28:02 +0200766 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
767 offset >> PAGE_SHIFT) {
Imre Deak2db76d72013-03-26 15:14:18 +0200768 struct page *page = sg_page_iter_page(&sg_iter);
Daniel Vetter58642882012-03-25 19:47:37 +0200769 int partial_cacheline_write;
Chris Wilsone5281cc2010-10-28 13:45:36 +0100770
Chris Wilson9da3da62012-06-01 15:20:22 +0100771 if (remain <= 0)
772 break;
773
Eric Anholt40123c12009-03-09 13:42:30 -0700774 /* Operation in this page
775 *
Eric Anholt40123c12009-03-09 13:42:30 -0700776 * shmem_page_offset = offset within page in shmem file
Eric Anholt40123c12009-03-09 13:42:30 -0700777 * page_length = bytes to copy for this page
778 */
Chris Wilsonc8cbbb82011-05-12 22:17:11 +0100779 shmem_page_offset = offset_in_page(offset);
Eric Anholt40123c12009-03-09 13:42:30 -0700780
781 page_length = remain;
782 if ((shmem_page_offset + page_length) > PAGE_SIZE)
783 page_length = PAGE_SIZE - shmem_page_offset;
Eric Anholt40123c12009-03-09 13:42:30 -0700784
Daniel Vetter58642882012-03-25 19:47:37 +0200785 /* If we don't overwrite a cacheline completely we need to be
786 * careful to have up-to-date data by first clflushing. Don't
787 * overcomplicate things and flush the entire patch. */
788 partial_cacheline_write = needs_clflush_before &&
789 ((shmem_page_offset | page_length)
790 & (boot_cpu_data.x86_clflush_size - 1));
791
Daniel Vetter8c599672011-12-14 13:57:31 +0100792 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
793 (page_to_phys(page) & (1 << 17)) != 0;
794
Daniel Vetterd174bd62012-03-25 19:47:40 +0200795 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
796 user_data, page_do_bit17_swizzling,
797 partial_cacheline_write,
798 needs_clflush_after);
799 if (ret == 0)
800 goto next_page;
Eric Anholt40123c12009-03-09 13:42:30 -0700801
Daniel Vettere244a442012-03-25 19:47:28 +0200802 hit_slowpath = 1;
Daniel Vettere244a442012-03-25 19:47:28 +0200803 mutex_unlock(&dev->struct_mutex);
Daniel Vetterd174bd62012-03-25 19:47:40 +0200804 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
805 user_data, page_do_bit17_swizzling,
806 partial_cacheline_write,
807 needs_clflush_after);
Eric Anholt40123c12009-03-09 13:42:30 -0700808
Daniel Vettere244a442012-03-25 19:47:28 +0200809 mutex_lock(&dev->struct_mutex);
Chris Wilson755d2212012-09-04 21:02:55 +0100810
Daniel Vettere244a442012-03-25 19:47:28 +0200811next_page:
Chris Wilsone5281cc2010-10-28 13:45:36 +0100812 set_page_dirty(page);
813 mark_page_accessed(page);
Chris Wilsone5281cc2010-10-28 13:45:36 +0100814
Chris Wilson755d2212012-09-04 21:02:55 +0100815 if (ret)
Daniel Vetter8c599672011-12-14 13:57:31 +0100816 goto out;
Daniel Vetter8c599672011-12-14 13:57:31 +0100817
Eric Anholt40123c12009-03-09 13:42:30 -0700818 remain -= page_length;
Daniel Vetter8c599672011-12-14 13:57:31 +0100819 user_data += page_length;
Eric Anholt40123c12009-03-09 13:42:30 -0700820 offset += page_length;
821 }
822
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100823out:
Chris Wilson755d2212012-09-04 21:02:55 +0100824 i915_gem_object_unpin_pages(obj);
825
Daniel Vettere244a442012-03-25 19:47:28 +0200826 if (hit_slowpath) {
Daniel Vetter8dcf0152012-11-15 16:53:58 +0100827 /*
828 * Fixup: Flush cpu caches in case we didn't flush the dirty
829 * cachelines in-line while writing and the object moved
830 * out of the cpu write domain while we've dropped the lock.
831 */
832 if (!needs_clflush_after &&
833 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
Daniel Vettere244a442012-03-25 19:47:28 +0200834 i915_gem_clflush_object(obj);
Ben Widawskye76e9ae2012-11-04 09:21:27 -0800835 i915_gem_chipset_flush(dev);
Daniel Vettere244a442012-03-25 19:47:28 +0200836 }
Daniel Vetter8c599672011-12-14 13:57:31 +0100837 }
Eric Anholt40123c12009-03-09 13:42:30 -0700838
Daniel Vetter58642882012-03-25 19:47:37 +0200839 if (needs_clflush_after)
Ben Widawskye76e9ae2012-11-04 09:21:27 -0800840 i915_gem_chipset_flush(dev);
Daniel Vetter58642882012-03-25 19:47:37 +0200841
Eric Anholt40123c12009-03-09 13:42:30 -0700842 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700843}
844
845/**
846 * Writes data to the object referenced by handle.
847 *
848 * On error, the contents of the buffer that were to be modified are undefined.
849 */
850int
851i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100852 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -0700853{
854 struct drm_i915_gem_pwrite *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +0000855 struct drm_i915_gem_object *obj;
Chris Wilson51311d02010-11-17 09:10:42 +0000856 int ret;
857
858 if (args->size == 0)
859 return 0;
860
861 if (!access_ok(VERIFY_READ,
Ville Syrjälä2bb46292013-02-22 16:12:51 +0200862 to_user_ptr(args->data_ptr),
Chris Wilson51311d02010-11-17 09:10:42 +0000863 args->size))
864 return -EFAULT;
865
Ville Syrjälä2bb46292013-02-22 16:12:51 +0200866 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
Daniel Vetterf56f8212012-03-25 19:47:41 +0200867 args->size);
Chris Wilson51311d02010-11-17 09:10:42 +0000868 if (ret)
869 return -EFAULT;
Eric Anholt673a3942008-07-30 12:06:12 -0700870
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100871 ret = i915_mutex_lock_interruptible(dev);
872 if (ret)
873 return ret;
874
Chris Wilson05394f32010-11-08 19:18:58 +0000875 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +0000876 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100877 ret = -ENOENT;
878 goto unlock;
879 }
Eric Anholt673a3942008-07-30 12:06:12 -0700880
Chris Wilson7dcd2492010-09-26 20:21:44 +0100881 /* Bounds check destination. */
Chris Wilson05394f32010-11-08 19:18:58 +0000882 if (args->offset > obj->base.size ||
883 args->size > obj->base.size - args->offset) {
Chris Wilsonce9d4192010-09-26 20:50:05 +0100884 ret = -EINVAL;
Chris Wilson35b62a82010-09-26 20:23:38 +0100885 goto out;
Chris Wilsonce9d4192010-09-26 20:50:05 +0100886 }
887
Daniel Vetter1286ff72012-05-10 15:25:09 +0200888 /* prime objects have no backing filp to GEM pread/pwrite
889 * pages from.
890 */
891 if (!obj->base.filp) {
892 ret = -EINVAL;
893 goto out;
894 }
895
Chris Wilsondb53a302011-02-03 11:57:46 +0000896 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
897
Daniel Vetter935aaa62012-03-25 19:47:35 +0200898 ret = -EFAULT;
Eric Anholt673a3942008-07-30 12:06:12 -0700899 /* We can only do the GTT pwrite on untiled buffers, as otherwise
900 * it would end up going through the fenced access, and we'll get
901 * different detiling behavior between reading and writing.
902 * pread/pwrite currently are reading and writing from the CPU
903 * perspective, requiring manual detiling by the client.
904 */
Daniel Vetter5c0480f2011-12-14 13:57:30 +0100905 if (obj->phys_obj) {
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100906 ret = i915_gem_phys_pwrite(dev, obj, args, file);
Daniel Vetter5c0480f2011-12-14 13:57:30 +0100907 goto out;
908 }
909
Chris Wilson86a1ee22012-08-11 15:41:04 +0100910 if (obj->cache_level == I915_CACHE_NONE &&
Daniel Vetterc07496f2012-04-13 15:51:51 +0200911 obj->tiling_mode == I915_TILING_NONE &&
Daniel Vetter5c0480f2011-12-14 13:57:30 +0100912 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100913 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
Daniel Vetter935aaa62012-03-25 19:47:35 +0200914 /* Note that the gtt paths might fail with non-page-backed user
915 * pointers (e.g. gtt mappings when moving data between
916 * textures). Fallback to the shmem path in that case. */
Eric Anholt40123c12009-03-09 13:42:30 -0700917 }
Eric Anholt673a3942008-07-30 12:06:12 -0700918
Chris Wilson86a1ee22012-08-11 15:41:04 +0100919 if (ret == -EFAULT || ret == -ENOSPC)
Daniel Vetter935aaa62012-03-25 19:47:35 +0200920 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
Daniel Vetter5c0480f2011-12-14 13:57:30 +0100921
Chris Wilson35b62a82010-09-26 20:23:38 +0100922out:
Chris Wilson05394f32010-11-08 19:18:58 +0000923 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +0100924unlock:
Chris Wilsonfbd5a262010-10-14 15:03:58 +0100925 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -0700926 return ret;
927}
928
Chris Wilsonb3612372012-08-24 09:35:08 +0100929int
Daniel Vetter33196de2012-11-14 17:14:05 +0100930i915_gem_check_wedge(struct i915_gpu_error *error,
Chris Wilsonb3612372012-08-24 09:35:08 +0100931 bool interruptible)
932{
Daniel Vetter1f83fee2012-11-15 17:17:22 +0100933 if (i915_reset_in_progress(error)) {
Chris Wilsonb3612372012-08-24 09:35:08 +0100934 /* Non-interruptible callers can't handle -EAGAIN, hence return
935 * -EIO unconditionally for these. */
936 if (!interruptible)
937 return -EIO;
938
Daniel Vetter1f83fee2012-11-15 17:17:22 +0100939 /* Recovery complete, but the reset failed ... */
940 if (i915_terminally_wedged(error))
Chris Wilsonb3612372012-08-24 09:35:08 +0100941 return -EIO;
942
943 return -EAGAIN;
944 }
945
946 return 0;
947}
948
949/*
950 * Compare seqno against outstanding lazy request. Emit a request if they are
951 * equal.
952 */
953static int
954i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
955{
956 int ret;
957
958 BUG_ON(!mutex_is_locked(&ring->dev->struct_mutex));
959
960 ret = 0;
961 if (seqno == ring->outstanding_lazy_request)
962 ret = i915_add_request(ring, NULL, NULL);
963
964 return ret;
965}
966
967/**
968 * __wait_seqno - wait until execution of seqno has finished
969 * @ring: the ring expected to report seqno
970 * @seqno: duh!
Daniel Vetterf69061b2012-12-06 09:01:42 +0100971 * @reset_counter: reset sequence associated with the given seqno
Chris Wilsonb3612372012-08-24 09:35:08 +0100972 * @interruptible: do an interruptible wait (normally yes)
973 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
974 *
Daniel Vetterf69061b2012-12-06 09:01:42 +0100975 * Note: It is of utmost importance that the passed in seqno and reset_counter
976 * values have been read by the caller in an smp safe manner. Where read-side
977 * locks are involved, it is sufficient to read the reset_counter before
978 * unlocking the lock that protects the seqno. For lockless tricks, the
979 * reset_counter _must_ be read before, and an appropriate smp_rmb must be
980 * inserted.
981 *
Chris Wilsonb3612372012-08-24 09:35:08 +0100982 * Returns 0 if the seqno was found within the alloted time. Else returns the
983 * errno with remaining time filled in timeout argument.
984 */
985static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
Daniel Vetterf69061b2012-12-06 09:01:42 +0100986 unsigned reset_counter,
Chris Wilsonb3612372012-08-24 09:35:08 +0100987 bool interruptible, struct timespec *timeout)
988{
989 drm_i915_private_t *dev_priv = ring->dev->dev_private;
990 struct timespec before, now, wait_time={1,0};
991 unsigned long timeout_jiffies;
992 long end;
993 bool wait_forever = true;
994 int ret;
995
996 if (i915_seqno_passed(ring->get_seqno(ring, true), seqno))
997 return 0;
998
999 trace_i915_gem_request_wait_begin(ring, seqno);
1000
1001 if (timeout != NULL) {
1002 wait_time = *timeout;
1003 wait_forever = false;
1004 }
1005
1006 timeout_jiffies = timespec_to_jiffies(&wait_time);
1007
1008 if (WARN_ON(!ring->irq_get(ring)))
1009 return -ENODEV;
1010
1011 /* Record current time in case interrupted by signal, or wedged * */
1012 getrawmonotonic(&before);
1013
1014#define EXIT_COND \
1015 (i915_seqno_passed(ring->get_seqno(ring, false), seqno) || \
Daniel Vetterf69061b2012-12-06 09:01:42 +01001016 i915_reset_in_progress(&dev_priv->gpu_error) || \
1017 reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
Chris Wilsonb3612372012-08-24 09:35:08 +01001018 do {
1019 if (interruptible)
1020 end = wait_event_interruptible_timeout(ring->irq_queue,
1021 EXIT_COND,
1022 timeout_jiffies);
1023 else
1024 end = wait_event_timeout(ring->irq_queue, EXIT_COND,
1025 timeout_jiffies);
1026
Daniel Vetterf69061b2012-12-06 09:01:42 +01001027 /* We need to check whether any gpu reset happened in between
1028 * the caller grabbing the seqno and now ... */
1029 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter))
1030 end = -EAGAIN;
1031
1032 /* ... but upgrade the -EGAIN to an -EIO if the gpu is truely
1033 * gone. */
Daniel Vetter33196de2012-11-14 17:14:05 +01001034 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
Chris Wilsonb3612372012-08-24 09:35:08 +01001035 if (ret)
1036 end = ret;
1037 } while (end == 0 && wait_forever);
1038
1039 getrawmonotonic(&now);
1040
1041 ring->irq_put(ring);
1042 trace_i915_gem_request_wait_end(ring, seqno);
1043#undef EXIT_COND
1044
1045 if (timeout) {
1046 struct timespec sleep_time = timespec_sub(now, before);
1047 *timeout = timespec_sub(*timeout, sleep_time);
Chris Wilson4f42f4e2013-04-26 16:22:46 +03001048 if (!timespec_valid(timeout)) /* i.e. negative time remains */
1049 set_normalized_timespec(timeout, 0, 0);
Chris Wilsonb3612372012-08-24 09:35:08 +01001050 }
1051
1052 switch (end) {
1053 case -EIO:
1054 case -EAGAIN: /* Wedged */
1055 case -ERESTARTSYS: /* Signal */
1056 return (int)end;
1057 case 0: /* Timeout */
Chris Wilsonb3612372012-08-24 09:35:08 +01001058 return -ETIME;
1059 default: /* Completed */
1060 WARN_ON(end < 0); /* We're not aware of other errors */
1061 return 0;
1062 }
1063}
1064
1065/**
1066 * Waits for a sequence number to be signaled, and cleans up the
1067 * request and object lists appropriately for that event.
1068 */
1069int
1070i915_wait_seqno(struct intel_ring_buffer *ring, uint32_t seqno)
1071{
1072 struct drm_device *dev = ring->dev;
1073 struct drm_i915_private *dev_priv = dev->dev_private;
1074 bool interruptible = dev_priv->mm.interruptible;
1075 int ret;
1076
1077 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1078 BUG_ON(seqno == 0);
1079
Daniel Vetter33196de2012-11-14 17:14:05 +01001080 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
Chris Wilsonb3612372012-08-24 09:35:08 +01001081 if (ret)
1082 return ret;
1083
1084 ret = i915_gem_check_olr(ring, seqno);
1085 if (ret)
1086 return ret;
1087
Daniel Vetterf69061b2012-12-06 09:01:42 +01001088 return __wait_seqno(ring, seqno,
1089 atomic_read(&dev_priv->gpu_error.reset_counter),
1090 interruptible, NULL);
Chris Wilsonb3612372012-08-24 09:35:08 +01001091}
1092
1093/**
1094 * Ensures that all rendering to the object has completed and the object is
1095 * safe to unbind from the GTT or access from the CPU.
1096 */
1097static __must_check int
1098i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1099 bool readonly)
1100{
1101 struct intel_ring_buffer *ring = obj->ring;
1102 u32 seqno;
1103 int ret;
1104
1105 seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1106 if (seqno == 0)
1107 return 0;
1108
1109 ret = i915_wait_seqno(ring, seqno);
1110 if (ret)
1111 return ret;
1112
1113 i915_gem_retire_requests_ring(ring);
1114
1115 /* Manually manage the write flush as we may have not yet
1116 * retired the buffer.
1117 */
1118 if (obj->last_write_seqno &&
1119 i915_seqno_passed(seqno, obj->last_write_seqno)) {
1120 obj->last_write_seqno = 0;
1121 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1122 }
1123
1124 return 0;
1125}
1126
Chris Wilson3236f572012-08-24 09:35:09 +01001127/* A nonblocking variant of the above wait. This is a highly dangerous routine
1128 * as the object state may change during this call.
1129 */
1130static __must_check int
1131i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1132 bool readonly)
1133{
1134 struct drm_device *dev = obj->base.dev;
1135 struct drm_i915_private *dev_priv = dev->dev_private;
1136 struct intel_ring_buffer *ring = obj->ring;
Daniel Vetterf69061b2012-12-06 09:01:42 +01001137 unsigned reset_counter;
Chris Wilson3236f572012-08-24 09:35:09 +01001138 u32 seqno;
1139 int ret;
1140
1141 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1142 BUG_ON(!dev_priv->mm.interruptible);
1143
1144 seqno = readonly ? obj->last_write_seqno : obj->last_read_seqno;
1145 if (seqno == 0)
1146 return 0;
1147
Daniel Vetter33196de2012-11-14 17:14:05 +01001148 ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
Chris Wilson3236f572012-08-24 09:35:09 +01001149 if (ret)
1150 return ret;
1151
1152 ret = i915_gem_check_olr(ring, seqno);
1153 if (ret)
1154 return ret;
1155
Daniel Vetterf69061b2012-12-06 09:01:42 +01001156 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
Chris Wilson3236f572012-08-24 09:35:09 +01001157 mutex_unlock(&dev->struct_mutex);
Daniel Vetterf69061b2012-12-06 09:01:42 +01001158 ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
Chris Wilson3236f572012-08-24 09:35:09 +01001159 mutex_lock(&dev->struct_mutex);
1160
1161 i915_gem_retire_requests_ring(ring);
1162
1163 /* Manually manage the write flush as we may have not yet
1164 * retired the buffer.
1165 */
1166 if (obj->last_write_seqno &&
1167 i915_seqno_passed(seqno, obj->last_write_seqno)) {
1168 obj->last_write_seqno = 0;
1169 obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
1170 }
1171
1172 return ret;
1173}
1174
Eric Anholt673a3942008-07-30 12:06:12 -07001175/**
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001176 * Called when user space prepares to use an object with the CPU, either
1177 * through the mmap ioctl's mapping or a GTT mapping.
Eric Anholt673a3942008-07-30 12:06:12 -07001178 */
1179int
1180i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001181 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001182{
1183 struct drm_i915_gem_set_domain *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00001184 struct drm_i915_gem_object *obj;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001185 uint32_t read_domains = args->read_domains;
1186 uint32_t write_domain = args->write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -07001187 int ret;
1188
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001189 /* Only handle setting domains to types used by the CPU. */
Chris Wilson21d509e2009-06-06 09:46:02 +01001190 if (write_domain & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001191 return -EINVAL;
1192
Chris Wilson21d509e2009-06-06 09:46:02 +01001193 if (read_domains & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001194 return -EINVAL;
1195
1196 /* Having something in the write domain implies it's in the read
1197 * domain, and only that read domain. Enforce that in the request.
1198 */
1199 if (write_domain != 0 && read_domains != write_domain)
1200 return -EINVAL;
1201
Chris Wilson76c1dec2010-09-25 11:22:51 +01001202 ret = i915_mutex_lock_interruptible(dev);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001203 if (ret)
Chris Wilson76c1dec2010-09-25 11:22:51 +01001204 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07001205
Chris Wilson05394f32010-11-08 19:18:58 +00001206 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00001207 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001208 ret = -ENOENT;
1209 goto unlock;
Chris Wilson76c1dec2010-09-25 11:22:51 +01001210 }
Jesse Barnes652c3932009-08-17 13:31:43 -07001211
Chris Wilson3236f572012-08-24 09:35:09 +01001212 /* Try to flush the object off the GPU without holding the lock.
1213 * We will repeat the flush holding the lock in the normal manner
1214 * to catch cases where we are gazumped.
1215 */
1216 ret = i915_gem_object_wait_rendering__nonblocking(obj, !write_domain);
1217 if (ret)
1218 goto unref;
1219
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001220 if (read_domains & I915_GEM_DOMAIN_GTT) {
1221 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
Eric Anholt02354392008-11-26 13:58:13 -08001222
1223 /* Silently promote "you're not bound, there was nothing to do"
1224 * to success, since the client was just asking us to
1225 * make sure everything was done.
1226 */
1227 if (ret == -EINVAL)
1228 ret = 0;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001229 } else {
Eric Anholte47c68e2008-11-14 13:35:19 -08001230 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001231 }
1232
Chris Wilson3236f572012-08-24 09:35:09 +01001233unref:
Chris Wilson05394f32010-11-08 19:18:58 +00001234 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001235unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07001236 mutex_unlock(&dev->struct_mutex);
1237 return ret;
1238}
1239
1240/**
1241 * Called when user space has done writes to this buffer
1242 */
1243int
1244i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001245 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001246{
1247 struct drm_i915_gem_sw_finish *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00001248 struct drm_i915_gem_object *obj;
Eric Anholt673a3942008-07-30 12:06:12 -07001249 int ret = 0;
1250
Chris Wilson76c1dec2010-09-25 11:22:51 +01001251 ret = i915_mutex_lock_interruptible(dev);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001252 if (ret)
Chris Wilson76c1dec2010-09-25 11:22:51 +01001253 return ret;
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001254
Chris Wilson05394f32010-11-08 19:18:58 +00001255 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00001256 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001257 ret = -ENOENT;
1258 goto unlock;
Eric Anholt673a3942008-07-30 12:06:12 -07001259 }
1260
Eric Anholt673a3942008-07-30 12:06:12 -07001261 /* Pinned buffers may be scanout, so flush the cache */
Chris Wilson05394f32010-11-08 19:18:58 +00001262 if (obj->pin_count)
Eric Anholte47c68e2008-11-14 13:35:19 -08001263 i915_gem_object_flush_cpu_write_domain(obj);
1264
Chris Wilson05394f32010-11-08 19:18:58 +00001265 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001266unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07001267 mutex_unlock(&dev->struct_mutex);
1268 return ret;
1269}
1270
1271/**
1272 * Maps the contents of an object, returning the address it is mapped
1273 * into.
1274 *
1275 * While the mapping holds a reference on the contents of the object, it doesn't
1276 * imply a ref on the object itself.
1277 */
1278int
1279i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001280 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001281{
1282 struct drm_i915_gem_mmap *args = data;
1283 struct drm_gem_object *obj;
Eric Anholt673a3942008-07-30 12:06:12 -07001284 unsigned long addr;
1285
Chris Wilson05394f32010-11-08 19:18:58 +00001286 obj = drm_gem_object_lookup(dev, file, args->handle);
Eric Anholt673a3942008-07-30 12:06:12 -07001287 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +01001288 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07001289
Daniel Vetter1286ff72012-05-10 15:25:09 +02001290 /* prime objects have no backing filp to GEM mmap
1291 * pages from.
1292 */
1293 if (!obj->filp) {
1294 drm_gem_object_unreference_unlocked(obj);
1295 return -EINVAL;
1296 }
1297
Linus Torvalds6be5ceb2012-04-20 17:13:58 -07001298 addr = vm_mmap(obj->filp, 0, args->size,
Eric Anholt673a3942008-07-30 12:06:12 -07001299 PROT_READ | PROT_WRITE, MAP_SHARED,
1300 args->offset);
Luca Barbieribc9025b2010-02-09 05:49:12 +00001301 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001302 if (IS_ERR((void *)addr))
1303 return addr;
1304
1305 args->addr_ptr = (uint64_t) addr;
1306
1307 return 0;
1308}
1309
Jesse Barnesde151cf2008-11-12 10:03:55 -08001310/**
1311 * i915_gem_fault - fault a page into the GTT
1312 * vma: VMA in question
1313 * vmf: fault info
1314 *
1315 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1316 * from userspace. The fault handler takes care of binding the object to
1317 * the GTT (if needed), allocating and programming a fence register (again,
1318 * only if needed based on whether the old reg is still valid or the object
1319 * is tiled) and inserting a new PTE into the faulting process.
1320 *
1321 * Note that the faulting process may involve evicting existing objects
1322 * from the GTT and/or fence registers to make room. So performance may
1323 * suffer if the GTT working set is large or there are few fence registers
1324 * left.
1325 */
1326int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1327{
Chris Wilson05394f32010-11-08 19:18:58 +00001328 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1329 struct drm_device *dev = obj->base.dev;
Chris Wilson7d1c4802010-08-07 21:45:03 +01001330 drm_i915_private_t *dev_priv = dev->dev_private;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001331 pgoff_t page_offset;
1332 unsigned long pfn;
1333 int ret = 0;
Jesse Barnes0f973f22009-01-26 17:10:45 -08001334 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001335
1336 /* We don't use vmf->pgoff since that has the fake offset */
1337 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1338 PAGE_SHIFT;
1339
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001340 ret = i915_mutex_lock_interruptible(dev);
1341 if (ret)
1342 goto out;
Chris Wilsona00b10c2010-09-24 21:15:47 +01001343
Chris Wilsondb53a302011-02-03 11:57:46 +00001344 trace_i915_gem_object_fault(obj, page_offset, true, write);
1345
Chris Wilsoneb119bd2012-12-16 12:43:36 +00001346 /* Access to snoopable pages through the GTT is incoherent. */
1347 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1348 ret = -EINVAL;
1349 goto unlock;
1350 }
1351
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001352 /* Now bind it into the GTT if needed */
Chris Wilsonc9839302012-11-20 10:45:17 +00001353 ret = i915_gem_object_pin(obj, 0, true, false);
Chris Wilsond9e86c02010-11-10 16:40:20 +00001354 if (ret)
1355 goto unlock;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001356
Chris Wilsonc9839302012-11-20 10:45:17 +00001357 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1358 if (ret)
1359 goto unpin;
1360
1361 ret = i915_gem_object_get_fence(obj);
1362 if (ret)
1363 goto unpin;
Chris Wilson7d1c4802010-08-07 21:45:03 +01001364
Chris Wilson6299f992010-11-24 12:23:44 +00001365 obj->fault_mappable = true;
1366
Ben Widawsky5d4545a2013-01-17 12:45:15 -08001367 pfn = ((dev_priv->gtt.mappable_base + obj->gtt_offset) >> PAGE_SHIFT) +
Jesse Barnesde151cf2008-11-12 10:03:55 -08001368 page_offset;
1369
1370 /* Finally, remap it using the new GTT offset */
1371 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
Chris Wilsonc9839302012-11-20 10:45:17 +00001372unpin:
1373 i915_gem_object_unpin(obj);
Chris Wilsonc7150892009-09-23 00:43:56 +01001374unlock:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001375 mutex_unlock(&dev->struct_mutex);
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001376out:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001377 switch (ret) {
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001378 case -EIO:
Daniel Vettera9340cc2012-07-04 22:18:42 +02001379 /* If this -EIO is due to a gpu hang, give the reset code a
1380 * chance to clean up the mess. Otherwise return the proper
1381 * SIGBUS. */
Daniel Vetter1f83fee2012-11-15 17:17:22 +01001382 if (i915_terminally_wedged(&dev_priv->gpu_error))
Daniel Vettera9340cc2012-07-04 22:18:42 +02001383 return VM_FAULT_SIGBUS;
Chris Wilson045e7692010-11-07 09:18:22 +00001384 case -EAGAIN:
Chris Wilsond9bc7e92011-02-07 13:09:31 +00001385 /* Give the error handler a chance to run and move the
1386 * objects off the GPU active list. Next time we service the
1387 * fault, we should be able to transition the page into the
1388 * GTT without touching the GPU (and so avoid further
1389 * EIO/EGAIN). If the GPU is wedged, then there is no issue
1390 * with coherency, just lost writes.
1391 */
Chris Wilson045e7692010-11-07 09:18:22 +00001392 set_need_resched();
Chris Wilsonc7150892009-09-23 00:43:56 +01001393 case 0:
1394 case -ERESTARTSYS:
Chris Wilsonbed636a2011-02-11 20:31:19 +00001395 case -EINTR:
Dmitry Rogozhkine79e0fe2012-10-03 17:15:26 +03001396 case -EBUSY:
1397 /*
1398 * EBUSY is ok: this just means that another thread
1399 * already did the job.
1400 */
Chris Wilsonc7150892009-09-23 00:43:56 +01001401 return VM_FAULT_NOPAGE;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001402 case -ENOMEM:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001403 return VM_FAULT_OOM;
Daniel Vettera7c2e1a2012-10-17 11:17:16 +02001404 case -ENOSPC:
1405 return VM_FAULT_SIGBUS;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001406 default:
Daniel Vettera7c2e1a2012-10-17 11:17:16 +02001407 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
Chris Wilsonc7150892009-09-23 00:43:56 +01001408 return VM_FAULT_SIGBUS;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001409 }
1410}
1411
1412/**
Chris Wilson901782b2009-07-10 08:18:50 +01001413 * i915_gem_release_mmap - remove physical page mappings
1414 * @obj: obj in question
1415 *
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02001416 * Preserve the reservation of the mmapping with the DRM core code, but
Chris Wilson901782b2009-07-10 08:18:50 +01001417 * relinquish ownership of the pages back to the system.
1418 *
1419 * It is vital that we remove the page mapping if we have mapped a tiled
1420 * object through the GTT and then lose the fence register due to
1421 * resource pressure. Similarly if the object has been moved out of the
1422 * aperture, than pages mapped into userspace must be revoked. Removing the
1423 * mapping will then trigger a page fault on the next user access, allowing
1424 * fixup by i915_gem_fault().
1425 */
Eric Anholtd05ca302009-07-10 13:02:26 -07001426void
Chris Wilson05394f32010-11-08 19:18:58 +00001427i915_gem_release_mmap(struct drm_i915_gem_object *obj)
Chris Wilson901782b2009-07-10 08:18:50 +01001428{
Chris Wilson6299f992010-11-24 12:23:44 +00001429 if (!obj->fault_mappable)
1430 return;
Chris Wilson901782b2009-07-10 08:18:50 +01001431
Chris Wilsonf6e47882011-03-20 21:09:12 +00001432 if (obj->base.dev->dev_mapping)
1433 unmap_mapping_range(obj->base.dev->dev_mapping,
1434 (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1435 obj->base.size, 1);
Daniel Vetterfb7d5162010-10-01 22:05:20 +02001436
Chris Wilson6299f992010-11-24 12:23:44 +00001437 obj->fault_mappable = false;
Chris Wilson901782b2009-07-10 08:18:50 +01001438}
1439
Imre Deak0fa87792013-01-07 21:47:35 +02001440uint32_t
Chris Wilsone28f8712011-07-18 13:11:49 -07001441i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
Chris Wilson92b88ae2010-11-09 11:47:32 +00001442{
Chris Wilsone28f8712011-07-18 13:11:49 -07001443 uint32_t gtt_size;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001444
1445 if (INTEL_INFO(dev)->gen >= 4 ||
Chris Wilsone28f8712011-07-18 13:11:49 -07001446 tiling_mode == I915_TILING_NONE)
1447 return size;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001448
1449 /* Previous chips need a power-of-two fence region when tiling */
1450 if (INTEL_INFO(dev)->gen == 3)
Chris Wilsone28f8712011-07-18 13:11:49 -07001451 gtt_size = 1024*1024;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001452 else
Chris Wilsone28f8712011-07-18 13:11:49 -07001453 gtt_size = 512*1024;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001454
Chris Wilsone28f8712011-07-18 13:11:49 -07001455 while (gtt_size < size)
1456 gtt_size <<= 1;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001457
Chris Wilsone28f8712011-07-18 13:11:49 -07001458 return gtt_size;
Chris Wilson92b88ae2010-11-09 11:47:32 +00001459}
1460
Jesse Barnesde151cf2008-11-12 10:03:55 -08001461/**
1462 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1463 * @obj: object to check
1464 *
1465 * Return the required GTT alignment for an object, taking into account
Daniel Vetter5e783302010-11-14 22:32:36 +01001466 * potential fence register mapping.
Jesse Barnesde151cf2008-11-12 10:03:55 -08001467 */
Imre Deakd8651102013-01-07 21:47:33 +02001468uint32_t
1469i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
1470 int tiling_mode, bool fenced)
Jesse Barnesde151cf2008-11-12 10:03:55 -08001471{
Jesse Barnesde151cf2008-11-12 10:03:55 -08001472 /*
1473 * Minimum alignment is 4k (GTT page size), but might be greater
1474 * if a fence register is needed for the object.
1475 */
Imre Deakd8651102013-01-07 21:47:33 +02001476 if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
Chris Wilsone28f8712011-07-18 13:11:49 -07001477 tiling_mode == I915_TILING_NONE)
Jesse Barnesde151cf2008-11-12 10:03:55 -08001478 return 4096;
1479
1480 /*
1481 * Previous chips need to be aligned to the size of the smallest
1482 * fence register that can contain the object.
1483 */
Chris Wilsone28f8712011-07-18 13:11:49 -07001484 return i915_gem_get_gtt_size(dev, size, tiling_mode);
Chris Wilsona00b10c2010-09-24 21:15:47 +01001485}
1486
Chris Wilsond8cb5082012-08-11 15:41:03 +01001487static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
1488{
1489 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1490 int ret;
1491
1492 if (obj->base.map_list.map)
1493 return 0;
1494
Daniel Vetterda494d72012-12-20 15:11:16 +01001495 dev_priv->mm.shrinker_no_lock_stealing = true;
1496
Chris Wilsond8cb5082012-08-11 15:41:03 +01001497 ret = drm_gem_create_mmap_offset(&obj->base);
1498 if (ret != -ENOSPC)
Daniel Vetterda494d72012-12-20 15:11:16 +01001499 goto out;
Chris Wilsond8cb5082012-08-11 15:41:03 +01001500
1501 /* Badly fragmented mmap space? The only way we can recover
1502 * space is by destroying unwanted objects. We can't randomly release
1503 * mmap_offsets as userspace expects them to be persistent for the
1504 * lifetime of the objects. The closest we can is to release the
1505 * offsets on purgeable objects by truncating it and marking it purged,
1506 * which prevents userspace from ever using that object again.
1507 */
1508 i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
1509 ret = drm_gem_create_mmap_offset(&obj->base);
1510 if (ret != -ENOSPC)
Daniel Vetterda494d72012-12-20 15:11:16 +01001511 goto out;
Chris Wilsond8cb5082012-08-11 15:41:03 +01001512
1513 i915_gem_shrink_all(dev_priv);
Daniel Vetterda494d72012-12-20 15:11:16 +01001514 ret = drm_gem_create_mmap_offset(&obj->base);
1515out:
1516 dev_priv->mm.shrinker_no_lock_stealing = false;
1517
1518 return ret;
Chris Wilsond8cb5082012-08-11 15:41:03 +01001519}
1520
1521static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
1522{
1523 if (!obj->base.map_list.map)
1524 return;
1525
1526 drm_gem_free_mmap_offset(&obj->base);
1527}
1528
Jesse Barnesde151cf2008-11-12 10:03:55 -08001529int
Dave Airlieff72145b2011-02-07 12:16:14 +10001530i915_gem_mmap_gtt(struct drm_file *file,
1531 struct drm_device *dev,
1532 uint32_t handle,
1533 uint64_t *offset)
Jesse Barnesde151cf2008-11-12 10:03:55 -08001534{
Chris Wilsonda761a62010-10-27 17:37:08 +01001535 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson05394f32010-11-08 19:18:58 +00001536 struct drm_i915_gem_object *obj;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001537 int ret;
1538
Chris Wilson76c1dec2010-09-25 11:22:51 +01001539 ret = i915_mutex_lock_interruptible(dev);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001540 if (ret)
Chris Wilson76c1dec2010-09-25 11:22:51 +01001541 return ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001542
Dave Airlieff72145b2011-02-07 12:16:14 +10001543 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00001544 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001545 ret = -ENOENT;
1546 goto unlock;
1547 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08001548
Ben Widawsky5d4545a2013-01-17 12:45:15 -08001549 if (obj->base.size > dev_priv->gtt.mappable_end) {
Chris Wilsonda761a62010-10-27 17:37:08 +01001550 ret = -E2BIG;
Eric Anholtff56b0b2011-10-31 23:16:21 -07001551 goto out;
Chris Wilsonda761a62010-10-27 17:37:08 +01001552 }
1553
Chris Wilson05394f32010-11-08 19:18:58 +00001554 if (obj->madv != I915_MADV_WILLNEED) {
Chris Wilsonab182822009-09-22 18:46:17 +01001555 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001556 ret = -EINVAL;
1557 goto out;
Chris Wilsonab182822009-09-22 18:46:17 +01001558 }
1559
Chris Wilsond8cb5082012-08-11 15:41:03 +01001560 ret = i915_gem_object_create_mmap_offset(obj);
1561 if (ret)
1562 goto out;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001563
Dave Airlieff72145b2011-02-07 12:16:14 +10001564 *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001565
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001566out:
Chris Wilson05394f32010-11-08 19:18:58 +00001567 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001568unlock:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001569 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001570 return ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001571}
1572
Dave Airlieff72145b2011-02-07 12:16:14 +10001573/**
1574 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1575 * @dev: DRM device
1576 * @data: GTT mapping ioctl data
1577 * @file: GEM object info
1578 *
1579 * Simply returns the fake offset to userspace so it can mmap it.
1580 * The mmap call will end up in drm_gem_mmap(), which will set things
1581 * up so we can get faults in the handler above.
1582 *
1583 * The fault handler will take care of binding the object into the GTT
1584 * (since it may have been evicted to make room for something), allocating
1585 * a fence register, and mapping the appropriate aperture address into
1586 * userspace.
1587 */
1588int
1589i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1590 struct drm_file *file)
1591{
1592 struct drm_i915_gem_mmap_gtt *args = data;
1593
Dave Airlieff72145b2011-02-07 12:16:14 +10001594 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1595}
1596
Daniel Vetter225067e2012-08-20 10:23:20 +02001597/* Immediately discard the backing storage */
1598static void
1599i915_gem_object_truncate(struct drm_i915_gem_object *obj)
Chris Wilsone5281cc2010-10-28 13:45:36 +01001600{
Chris Wilsone5281cc2010-10-28 13:45:36 +01001601 struct inode *inode;
Chris Wilsone5281cc2010-10-28 13:45:36 +01001602
Chris Wilson4d6294bf2012-08-11 15:41:05 +01001603 i915_gem_object_free_mmap_offset(obj);
Daniel Vetter1286ff72012-05-10 15:25:09 +02001604
Chris Wilson4d6294bf2012-08-11 15:41:05 +01001605 if (obj->base.filp == NULL)
1606 return;
1607
Daniel Vetter225067e2012-08-20 10:23:20 +02001608 /* Our goal here is to return as much of the memory as
1609 * is possible back to the system as we are called from OOM.
1610 * To do this we must instruct the shmfs to drop all of its
1611 * backing pages, *now*.
Chris Wilsone5281cc2010-10-28 13:45:36 +01001612 */
Al Viro496ad9a2013-01-23 17:07:38 -05001613 inode = file_inode(obj->base.filp);
Daniel Vetter225067e2012-08-20 10:23:20 +02001614 shmem_truncate_range(inode, 0, (loff_t)-1);
Hugh Dickins5949eac2011-06-27 16:18:18 -07001615
Daniel Vetter225067e2012-08-20 10:23:20 +02001616 obj->madv = __I915_MADV_PURGED;
Chris Wilsone5281cc2010-10-28 13:45:36 +01001617}
Chris Wilsone5281cc2010-10-28 13:45:36 +01001618
Daniel Vetter225067e2012-08-20 10:23:20 +02001619static inline int
1620i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1621{
1622 return obj->madv == I915_MADV_DONTNEED;
Chris Wilsone5281cc2010-10-28 13:45:36 +01001623}
1624
Chris Wilson5cdf5882010-09-27 15:51:07 +01001625static void
Chris Wilson05394f32010-11-08 19:18:58 +00001626i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07001627{
Imre Deak90797e62013-02-18 19:28:03 +02001628 struct sg_page_iter sg_iter;
1629 int ret;
Daniel Vetter1286ff72012-05-10 15:25:09 +02001630
Chris Wilson05394f32010-11-08 19:18:58 +00001631 BUG_ON(obj->madv == __I915_MADV_PURGED);
Eric Anholt856fa192009-03-19 14:10:50 -07001632
Chris Wilson6c085a72012-08-20 11:40:46 +02001633 ret = i915_gem_object_set_to_cpu_domain(obj, true);
1634 if (ret) {
1635 /* In the event of a disaster, abandon all caches and
1636 * hope for the best.
1637 */
1638 WARN_ON(ret != -EIO);
1639 i915_gem_clflush_object(obj);
1640 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
1641 }
1642
Daniel Vetter6dacfd22011-09-12 21:30:02 +02001643 if (i915_gem_object_needs_bit17_swizzle(obj))
Eric Anholt280b7132009-03-12 16:56:27 -07001644 i915_gem_object_save_bit_17_swizzle(obj);
1645
Chris Wilson05394f32010-11-08 19:18:58 +00001646 if (obj->madv == I915_MADV_DONTNEED)
1647 obj->dirty = 0;
Chris Wilson3ef94da2009-09-14 16:50:29 +01001648
Imre Deak90797e62013-02-18 19:28:03 +02001649 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
Imre Deak2db76d72013-03-26 15:14:18 +02001650 struct page *page = sg_page_iter_page(&sg_iter);
Chris Wilson9da3da62012-06-01 15:20:22 +01001651
Chris Wilson05394f32010-11-08 19:18:58 +00001652 if (obj->dirty)
Chris Wilson9da3da62012-06-01 15:20:22 +01001653 set_page_dirty(page);
Chris Wilson3ef94da2009-09-14 16:50:29 +01001654
Chris Wilson05394f32010-11-08 19:18:58 +00001655 if (obj->madv == I915_MADV_WILLNEED)
Chris Wilson9da3da62012-06-01 15:20:22 +01001656 mark_page_accessed(page);
Chris Wilson3ef94da2009-09-14 16:50:29 +01001657
Chris Wilson9da3da62012-06-01 15:20:22 +01001658 page_cache_release(page);
Chris Wilson3ef94da2009-09-14 16:50:29 +01001659 }
Chris Wilson05394f32010-11-08 19:18:58 +00001660 obj->dirty = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07001661
Chris Wilson9da3da62012-06-01 15:20:22 +01001662 sg_free_table(obj->pages);
1663 kfree(obj->pages);
Chris Wilson37e680a2012-06-07 15:38:42 +01001664}
1665
Chris Wilsondd624af2013-01-15 12:39:35 +00001666int
Chris Wilson37e680a2012-06-07 15:38:42 +01001667i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
1668{
1669 const struct drm_i915_gem_object_ops *ops = obj->ops;
1670
Chris Wilson2f745ad2012-09-04 21:02:58 +01001671 if (obj->pages == NULL)
Chris Wilson37e680a2012-06-07 15:38:42 +01001672 return 0;
1673
1674 BUG_ON(obj->gtt_space);
1675
Chris Wilsona5570172012-09-04 21:02:54 +01001676 if (obj->pages_pin_count)
1677 return -EBUSY;
1678
Chris Wilsona2165e32012-12-03 11:49:00 +00001679 /* ->put_pages might need to allocate memory for the bit17 swizzle
1680 * array, hence protect them from being reaped by removing them from gtt
1681 * lists early. */
1682 list_del(&obj->gtt_list);
1683
Chris Wilson37e680a2012-06-07 15:38:42 +01001684 ops->put_pages(obj);
Chris Wilson05394f32010-11-08 19:18:58 +00001685 obj->pages = NULL;
Chris Wilson6c085a72012-08-20 11:40:46 +02001686
Chris Wilson6c085a72012-08-20 11:40:46 +02001687 if (i915_gem_object_is_purgeable(obj))
1688 i915_gem_object_truncate(obj);
1689
1690 return 0;
1691}
1692
1693static long
Daniel Vetter93927ca2013-01-10 18:03:00 +01001694__i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
1695 bool purgeable_only)
Chris Wilson6c085a72012-08-20 11:40:46 +02001696{
1697 struct drm_i915_gem_object *obj, *next;
1698 long count = 0;
1699
1700 list_for_each_entry_safe(obj, next,
1701 &dev_priv->mm.unbound_list,
1702 gtt_list) {
Daniel Vetter93927ca2013-01-10 18:03:00 +01001703 if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
Chris Wilson37e680a2012-06-07 15:38:42 +01001704 i915_gem_object_put_pages(obj) == 0) {
Chris Wilson6c085a72012-08-20 11:40:46 +02001705 count += obj->base.size >> PAGE_SHIFT;
1706 if (count >= target)
1707 return count;
1708 }
1709 }
1710
1711 list_for_each_entry_safe(obj, next,
1712 &dev_priv->mm.inactive_list,
1713 mm_list) {
Daniel Vetter93927ca2013-01-10 18:03:00 +01001714 if ((i915_gem_object_is_purgeable(obj) || !purgeable_only) &&
Chris Wilson6c085a72012-08-20 11:40:46 +02001715 i915_gem_object_unbind(obj) == 0 &&
Chris Wilson37e680a2012-06-07 15:38:42 +01001716 i915_gem_object_put_pages(obj) == 0) {
Chris Wilson6c085a72012-08-20 11:40:46 +02001717 count += obj->base.size >> PAGE_SHIFT;
1718 if (count >= target)
1719 return count;
1720 }
1721 }
1722
1723 return count;
1724}
1725
Daniel Vetter93927ca2013-01-10 18:03:00 +01001726static long
1727i915_gem_purge(struct drm_i915_private *dev_priv, long target)
1728{
1729 return __i915_gem_shrink(dev_priv, target, true);
1730}
1731
Chris Wilson6c085a72012-08-20 11:40:46 +02001732static void
1733i915_gem_shrink_all(struct drm_i915_private *dev_priv)
1734{
1735 struct drm_i915_gem_object *obj, *next;
1736
1737 i915_gem_evict_everything(dev_priv->dev);
1738
1739 list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list, gtt_list)
Chris Wilson37e680a2012-06-07 15:38:42 +01001740 i915_gem_object_put_pages(obj);
Daniel Vetter225067e2012-08-20 10:23:20 +02001741}
1742
Chris Wilson37e680a2012-06-07 15:38:42 +01001743static int
Chris Wilson6c085a72012-08-20 11:40:46 +02001744i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07001745{
Chris Wilson6c085a72012-08-20 11:40:46 +02001746 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07001747 int page_count, i;
1748 struct address_space *mapping;
Chris Wilson9da3da62012-06-01 15:20:22 +01001749 struct sg_table *st;
1750 struct scatterlist *sg;
Imre Deak90797e62013-02-18 19:28:03 +02001751 struct sg_page_iter sg_iter;
Eric Anholt673a3942008-07-30 12:06:12 -07001752 struct page *page;
Imre Deak90797e62013-02-18 19:28:03 +02001753 unsigned long last_pfn = 0; /* suppress gcc warning */
Chris Wilson6c085a72012-08-20 11:40:46 +02001754 gfp_t gfp;
Eric Anholt673a3942008-07-30 12:06:12 -07001755
Chris Wilson6c085a72012-08-20 11:40:46 +02001756 /* Assert that the object is not currently in any GPU domain. As it
1757 * wasn't in the GTT, there shouldn't be any way it could have been in
1758 * a GPU cache
1759 */
1760 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
1761 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
1762
Chris Wilson9da3da62012-06-01 15:20:22 +01001763 st = kmalloc(sizeof(*st), GFP_KERNEL);
1764 if (st == NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07001765 return -ENOMEM;
1766
Chris Wilson9da3da62012-06-01 15:20:22 +01001767 page_count = obj->base.size / PAGE_SIZE;
1768 if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
1769 sg_free_table(st);
1770 kfree(st);
1771 return -ENOMEM;
1772 }
1773
1774 /* Get the list of pages out of our struct file. They'll be pinned
1775 * at this point until we release them.
1776 *
1777 * Fail silently without starting the shrinker
1778 */
Al Viro496ad9a2013-01-23 17:07:38 -05001779 mapping = file_inode(obj->base.filp)->i_mapping;
Chris Wilson6c085a72012-08-20 11:40:46 +02001780 gfp = mapping_gfp_mask(mapping);
Linus Torvaldscaf49192012-12-10 10:51:16 -08001781 gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
Chris Wilson6c085a72012-08-20 11:40:46 +02001782 gfp &= ~(__GFP_IO | __GFP_WAIT);
Imre Deak90797e62013-02-18 19:28:03 +02001783 sg = st->sgl;
1784 st->nents = 0;
1785 for (i = 0; i < page_count; i++) {
Chris Wilson6c085a72012-08-20 11:40:46 +02001786 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1787 if (IS_ERR(page)) {
1788 i915_gem_purge(dev_priv, page_count);
1789 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1790 }
1791 if (IS_ERR(page)) {
1792 /* We've tried hard to allocate the memory by reaping
1793 * our own buffer, now let the real VM do its job and
1794 * go down in flames if truly OOM.
1795 */
Linus Torvaldscaf49192012-12-10 10:51:16 -08001796 gfp &= ~(__GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD);
Chris Wilson6c085a72012-08-20 11:40:46 +02001797 gfp |= __GFP_IO | __GFP_WAIT;
1798
1799 i915_gem_shrink_all(dev_priv);
1800 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
1801 if (IS_ERR(page))
1802 goto err_pages;
1803
Linus Torvaldscaf49192012-12-10 10:51:16 -08001804 gfp |= __GFP_NORETRY | __GFP_NOWARN | __GFP_NO_KSWAPD;
Chris Wilson6c085a72012-08-20 11:40:46 +02001805 gfp &= ~(__GFP_IO | __GFP_WAIT);
1806 }
Eric Anholt673a3942008-07-30 12:06:12 -07001807
Imre Deak90797e62013-02-18 19:28:03 +02001808 if (!i || page_to_pfn(page) != last_pfn + 1) {
1809 if (i)
1810 sg = sg_next(sg);
1811 st->nents++;
1812 sg_set_page(sg, page, PAGE_SIZE, 0);
1813 } else {
1814 sg->length += PAGE_SIZE;
1815 }
1816 last_pfn = page_to_pfn(page);
Eric Anholt673a3942008-07-30 12:06:12 -07001817 }
1818
Imre Deak90797e62013-02-18 19:28:03 +02001819 sg_mark_end(sg);
Chris Wilson74ce6b62012-10-19 15:51:06 +01001820 obj->pages = st;
1821
Eric Anholt673a3942008-07-30 12:06:12 -07001822 if (i915_gem_object_needs_bit17_swizzle(obj))
1823 i915_gem_object_do_bit_17_swizzle(obj);
1824
1825 return 0;
1826
1827err_pages:
Imre Deak90797e62013-02-18 19:28:03 +02001828 sg_mark_end(sg);
1829 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
Imre Deak2db76d72013-03-26 15:14:18 +02001830 page_cache_release(sg_page_iter_page(&sg_iter));
Chris Wilson9da3da62012-06-01 15:20:22 +01001831 sg_free_table(st);
1832 kfree(st);
Eric Anholt673a3942008-07-30 12:06:12 -07001833 return PTR_ERR(page);
Eric Anholt673a3942008-07-30 12:06:12 -07001834}
1835
Chris Wilson37e680a2012-06-07 15:38:42 +01001836/* Ensure that the associated pages are gathered from the backing storage
1837 * and pinned into our object. i915_gem_object_get_pages() may be called
1838 * multiple times before they are released by a single call to
1839 * i915_gem_object_put_pages() - once the pages are no longer referenced
1840 * either as a result of memory pressure (reaping pages under the shrinker)
1841 * or as the object is itself released.
1842 */
1843int
1844i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
1845{
1846 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
1847 const struct drm_i915_gem_object_ops *ops = obj->ops;
1848 int ret;
1849
Chris Wilson2f745ad2012-09-04 21:02:58 +01001850 if (obj->pages)
Chris Wilson37e680a2012-06-07 15:38:42 +01001851 return 0;
1852
Chris Wilson43e28f02013-01-08 10:53:09 +00001853 if (obj->madv != I915_MADV_WILLNEED) {
1854 DRM_ERROR("Attempting to obtain a purgeable object\n");
1855 return -EINVAL;
1856 }
1857
Chris Wilsona5570172012-09-04 21:02:54 +01001858 BUG_ON(obj->pages_pin_count);
1859
Chris Wilson37e680a2012-06-07 15:38:42 +01001860 ret = ops->get_pages(obj);
1861 if (ret)
1862 return ret;
1863
1864 list_add_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
1865 return 0;
Eric Anholt673a3942008-07-30 12:06:12 -07001866}
1867
Chris Wilson54cf91d2010-11-25 18:00:26 +00001868void
Chris Wilson05394f32010-11-08 19:18:58 +00001869i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
Chris Wilson9d7730912012-11-27 16:22:52 +00001870 struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001871{
Chris Wilson05394f32010-11-08 19:18:58 +00001872 struct drm_device *dev = obj->base.dev;
Chris Wilson69dc4982010-10-19 10:36:51 +01001873 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson9d7730912012-11-27 16:22:52 +00001874 u32 seqno = intel_ring_get_seqno(ring);
Daniel Vetter617dbe22010-02-11 22:16:02 +01001875
Zou Nan hai852835f2010-05-21 09:08:56 +08001876 BUG_ON(ring == NULL);
Chris Wilson05394f32010-11-08 19:18:58 +00001877 obj->ring = ring;
Eric Anholt673a3942008-07-30 12:06:12 -07001878
1879 /* Add a reference if we're newly entering the active list. */
Chris Wilson05394f32010-11-08 19:18:58 +00001880 if (!obj->active) {
1881 drm_gem_object_reference(&obj->base);
1882 obj->active = 1;
Eric Anholt673a3942008-07-30 12:06:12 -07001883 }
Daniel Vettere35a41d2010-02-11 22:13:59 +01001884
Eric Anholt673a3942008-07-30 12:06:12 -07001885 /* Move from whatever list we were on to the tail of execution. */
Chris Wilson05394f32010-11-08 19:18:58 +00001886 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1887 list_move_tail(&obj->ring_list, &ring->active_list);
Chris Wilsoncaea7472010-11-12 13:53:37 +00001888
Chris Wilson0201f1e2012-07-20 12:41:01 +01001889 obj->last_read_seqno = seqno;
Chris Wilson7dd49062012-03-21 10:48:18 +00001890
Chris Wilsoncaea7472010-11-12 13:53:37 +00001891 if (obj->fenced_gpu_access) {
Chris Wilsoncaea7472010-11-12 13:53:37 +00001892 obj->last_fenced_seqno = seqno;
Chris Wilsoncaea7472010-11-12 13:53:37 +00001893
Chris Wilson7dd49062012-03-21 10:48:18 +00001894 /* Bump MRU to take account of the delayed flush */
1895 if (obj->fence_reg != I915_FENCE_REG_NONE) {
1896 struct drm_i915_fence_reg *reg;
1897
1898 reg = &dev_priv->fence_regs[obj->fence_reg];
1899 list_move_tail(&reg->lru_list,
1900 &dev_priv->mm.fence_list);
1901 }
Chris Wilsoncaea7472010-11-12 13:53:37 +00001902 }
1903}
1904
1905static void
Chris Wilsoncaea7472010-11-12 13:53:37 +00001906i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1907{
1908 struct drm_device *dev = obj->base.dev;
1909 struct drm_i915_private *dev_priv = dev->dev_private;
1910
Chris Wilson65ce3022012-07-20 12:41:02 +01001911 BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
Chris Wilsoncaea7472010-11-12 13:53:37 +00001912 BUG_ON(!obj->active);
Chris Wilson65ce3022012-07-20 12:41:02 +01001913
Chris Wilsoncaea7472010-11-12 13:53:37 +00001914 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1915
Chris Wilson65ce3022012-07-20 12:41:02 +01001916 list_del_init(&obj->ring_list);
Chris Wilsoncaea7472010-11-12 13:53:37 +00001917 obj->ring = NULL;
1918
Chris Wilson65ce3022012-07-20 12:41:02 +01001919 obj->last_read_seqno = 0;
1920 obj->last_write_seqno = 0;
1921 obj->base.write_domain = 0;
1922
1923 obj->last_fenced_seqno = 0;
Chris Wilsoncaea7472010-11-12 13:53:37 +00001924 obj->fenced_gpu_access = false;
Chris Wilsoncaea7472010-11-12 13:53:37 +00001925
1926 obj->active = 0;
1927 drm_gem_object_unreference(&obj->base);
1928
1929 WARN_ON(i915_verify_lists(dev));
Eric Anholtce44b0e2008-11-06 16:00:31 -08001930}
Eric Anholt673a3942008-07-30 12:06:12 -07001931
Chris Wilson9d7730912012-11-27 16:22:52 +00001932static int
Mika Kuoppalafca26bb2012-12-19 11:13:08 +02001933i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
Daniel Vetter53d227f2012-01-25 16:32:49 +01001934{
Chris Wilson9d7730912012-11-27 16:22:52 +00001935 struct drm_i915_private *dev_priv = dev->dev_private;
1936 struct intel_ring_buffer *ring;
1937 int ret, i, j;
Daniel Vetter53d227f2012-01-25 16:32:49 +01001938
Chris Wilson107f27a52012-12-10 13:56:17 +02001939 /* Carefully retire all requests without writing to the rings */
Chris Wilson9d7730912012-11-27 16:22:52 +00001940 for_each_ring(ring, dev_priv, i) {
Chris Wilson107f27a52012-12-10 13:56:17 +02001941 ret = intel_ring_idle(ring);
1942 if (ret)
1943 return ret;
Chris Wilson9d7730912012-11-27 16:22:52 +00001944 }
Chris Wilson9d7730912012-11-27 16:22:52 +00001945 i915_gem_retire_requests(dev);
Chris Wilson107f27a52012-12-10 13:56:17 +02001946
1947 /* Finally reset hw state */
Chris Wilson9d7730912012-11-27 16:22:52 +00001948 for_each_ring(ring, dev_priv, i) {
Mika Kuoppalafca26bb2012-12-19 11:13:08 +02001949 intel_ring_init_seqno(ring, seqno);
Mika Kuoppala498d2ac2012-12-04 15:12:04 +02001950
Chris Wilson9d7730912012-11-27 16:22:52 +00001951 for (j = 0; j < ARRAY_SIZE(ring->sync_seqno); j++)
1952 ring->sync_seqno[j] = 0;
1953 }
1954
1955 return 0;
Daniel Vetter53d227f2012-01-25 16:32:49 +01001956}
1957
Mika Kuoppalafca26bb2012-12-19 11:13:08 +02001958int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
1959{
1960 struct drm_i915_private *dev_priv = dev->dev_private;
1961 int ret;
1962
1963 if (seqno == 0)
1964 return -EINVAL;
1965
1966 /* HWS page needs to be set less than what we
1967 * will inject to ring
1968 */
1969 ret = i915_gem_init_seqno(dev, seqno - 1);
1970 if (ret)
1971 return ret;
1972
1973 /* Carefully set the last_seqno value so that wrap
1974 * detection still works
1975 */
1976 dev_priv->next_seqno = seqno;
1977 dev_priv->last_seqno = seqno - 1;
1978 if (dev_priv->last_seqno == 0)
1979 dev_priv->last_seqno--;
1980
1981 return 0;
1982}
1983
Chris Wilson9d7730912012-11-27 16:22:52 +00001984int
1985i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
Daniel Vetter53d227f2012-01-25 16:32:49 +01001986{
Chris Wilson9d7730912012-11-27 16:22:52 +00001987 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter53d227f2012-01-25 16:32:49 +01001988
Chris Wilson9d7730912012-11-27 16:22:52 +00001989 /* reserve 0 for non-seqno */
1990 if (dev_priv->next_seqno == 0) {
Mika Kuoppalafca26bb2012-12-19 11:13:08 +02001991 int ret = i915_gem_init_seqno(dev, 0);
Chris Wilson9d7730912012-11-27 16:22:52 +00001992 if (ret)
1993 return ret;
1994
1995 dev_priv->next_seqno = 1;
1996 }
1997
Mika Kuoppalaf72b3432012-12-10 15:41:48 +02001998 *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
Chris Wilson9d7730912012-11-27 16:22:52 +00001999 return 0;
Daniel Vetter53d227f2012-01-25 16:32:49 +01002000}
2001
Chris Wilson3cce4692010-10-27 16:11:02 +01002002int
Chris Wilsondb53a302011-02-03 11:57:46 +00002003i915_add_request(struct intel_ring_buffer *ring,
Chris Wilsonf787a5f2010-09-24 16:02:42 +01002004 struct drm_file *file,
Chris Wilsonacb868d2012-09-26 13:47:30 +01002005 u32 *out_seqno)
Eric Anholt673a3942008-07-30 12:06:12 -07002006{
Chris Wilsondb53a302011-02-03 11:57:46 +00002007 drm_i915_private_t *dev_priv = ring->dev->dev_private;
Chris Wilsonacb868d2012-09-26 13:47:30 +01002008 struct drm_i915_gem_request *request;
Chris Wilsona71d8d92012-02-15 11:25:36 +00002009 u32 request_ring_position;
Eric Anholt673a3942008-07-30 12:06:12 -07002010 int was_empty;
Chris Wilson3cce4692010-10-27 16:11:02 +01002011 int ret;
2012
Daniel Vettercc889e02012-06-13 20:45:19 +02002013 /*
2014 * Emit any outstanding flushes - execbuf can fail to emit the flush
2015 * after having emitted the batchbuffer command. Hence we need to fix
2016 * things up similar to emitting the lazy request. The difference here
2017 * is that the flush _must_ happen before the next request, no matter
2018 * what.
2019 */
Chris Wilsona7b97612012-07-20 12:41:08 +01002020 ret = intel_ring_flush_all_caches(ring);
2021 if (ret)
2022 return ret;
Daniel Vettercc889e02012-06-13 20:45:19 +02002023
Chris Wilsonacb868d2012-09-26 13:47:30 +01002024 request = kmalloc(sizeof(*request), GFP_KERNEL);
2025 if (request == NULL)
2026 return -ENOMEM;
Daniel Vettercc889e02012-06-13 20:45:19 +02002027
Eric Anholt673a3942008-07-30 12:06:12 -07002028
Chris Wilsona71d8d92012-02-15 11:25:36 +00002029 /* Record the position of the start of the request so that
2030 * should we detect the updated seqno part-way through the
2031 * GPU processing the request, we never over-estimate the
2032 * position of the head.
2033 */
2034 request_ring_position = intel_ring_get_tail(ring);
2035
Chris Wilson9d7730912012-11-27 16:22:52 +00002036 ret = ring->add_request(ring);
Chris Wilson3bb73ab2012-07-20 12:40:59 +01002037 if (ret) {
2038 kfree(request);
2039 return ret;
2040 }
Eric Anholt673a3942008-07-30 12:06:12 -07002041
Chris Wilson9d7730912012-11-27 16:22:52 +00002042 request->seqno = intel_ring_get_seqno(ring);
Zou Nan hai852835f2010-05-21 09:08:56 +08002043 request->ring = ring;
Chris Wilsona71d8d92012-02-15 11:25:36 +00002044 request->tail = request_ring_position;
Mika Kuoppala0e50e962013-05-02 16:48:08 +03002045 request->ctx = ring->last_context;
2046
2047 if (request->ctx)
2048 i915_gem_context_reference(request->ctx);
2049
Eric Anholt673a3942008-07-30 12:06:12 -07002050 request->emitted_jiffies = jiffies;
Zou Nan hai852835f2010-05-21 09:08:56 +08002051 was_empty = list_empty(&ring->request_list);
2052 list_add_tail(&request->list, &ring->request_list);
Chris Wilson3bb73ab2012-07-20 12:40:59 +01002053 request->file_priv = NULL;
Zou Nan hai852835f2010-05-21 09:08:56 +08002054
Chris Wilsondb53a302011-02-03 11:57:46 +00002055 if (file) {
2056 struct drm_i915_file_private *file_priv = file->driver_priv;
2057
Chris Wilson1c255952010-09-26 11:03:27 +01002058 spin_lock(&file_priv->mm.lock);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01002059 request->file_priv = file_priv;
Eric Anholtb9624422009-06-03 07:27:35 +00002060 list_add_tail(&request->client_list,
Chris Wilsonf787a5f2010-09-24 16:02:42 +01002061 &file_priv->mm.request_list);
Chris Wilson1c255952010-09-26 11:03:27 +01002062 spin_unlock(&file_priv->mm.lock);
Eric Anholtb9624422009-06-03 07:27:35 +00002063 }
Eric Anholt673a3942008-07-30 12:06:12 -07002064
Chris Wilson9d7730912012-11-27 16:22:52 +00002065 trace_i915_gem_request_add(ring, request->seqno);
Daniel Vetter5391d0c2012-01-25 14:03:57 +01002066 ring->outstanding_lazy_request = 0;
Chris Wilsondb53a302011-02-03 11:57:46 +00002067
Ben Gamarif65d9422009-09-14 17:48:44 -04002068 if (!dev_priv->mm.suspended) {
Ben Widawsky3e0dc6b2011-06-29 10:26:42 -07002069 if (i915_enable_hangcheck) {
Daniel Vetter99584db2012-11-14 17:14:04 +01002070 mod_timer(&dev_priv->gpu_error.hangcheck_timer,
Chris Wilsoncecc21f2012-10-05 17:02:56 +01002071 round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES));
Ben Widawsky3e0dc6b2011-06-29 10:26:42 -07002072 }
Chris Wilsonf047e392012-07-21 12:31:41 +01002073 if (was_empty) {
Chris Wilsonb3b079d2010-09-13 23:44:34 +01002074 queue_delayed_work(dev_priv->wq,
Chris Wilsonbcb45082012-10-05 17:02:57 +01002075 &dev_priv->mm.retire_work,
2076 round_jiffies_up_relative(HZ));
Chris Wilsonf047e392012-07-21 12:31:41 +01002077 intel_mark_busy(dev_priv->dev);
2078 }
Ben Gamarif65d9422009-09-14 17:48:44 -04002079 }
Daniel Vettercc889e02012-06-13 20:45:19 +02002080
Chris Wilsonacb868d2012-09-26 13:47:30 +01002081 if (out_seqno)
Chris Wilson9d7730912012-11-27 16:22:52 +00002082 *out_seqno = request->seqno;
Chris Wilson3cce4692010-10-27 16:11:02 +01002083 return 0;
Eric Anholt673a3942008-07-30 12:06:12 -07002084}
2085
Chris Wilsonf787a5f2010-09-24 16:02:42 +01002086static inline void
2087i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
Eric Anholt673a3942008-07-30 12:06:12 -07002088{
Chris Wilson1c255952010-09-26 11:03:27 +01002089 struct drm_i915_file_private *file_priv = request->file_priv;
Eric Anholt673a3942008-07-30 12:06:12 -07002090
Chris Wilson1c255952010-09-26 11:03:27 +01002091 if (!file_priv)
2092 return;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002093
Chris Wilson1c255952010-09-26 11:03:27 +01002094 spin_lock(&file_priv->mm.lock);
Herton Ronaldo Krzesinski09bfa512011-03-17 13:45:12 +00002095 if (request->file_priv) {
2096 list_del(&request->client_list);
2097 request->file_priv = NULL;
2098 }
Chris Wilson1c255952010-09-26 11:03:27 +01002099 spin_unlock(&file_priv->mm.lock);
Eric Anholt673a3942008-07-30 12:06:12 -07002100}
2101
Mika Kuoppala0e50e962013-05-02 16:48:08 +03002102static void i915_gem_free_request(struct drm_i915_gem_request *request)
2103{
2104 list_del(&request->list);
2105 i915_gem_request_remove_from_client(request);
2106
2107 if (request->ctx)
2108 i915_gem_context_unreference(request->ctx);
2109
2110 kfree(request);
2111}
2112
Chris Wilsondfaae392010-09-22 10:31:52 +01002113static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
2114 struct intel_ring_buffer *ring)
Chris Wilson9375e442010-09-19 12:21:28 +01002115{
Chris Wilsondfaae392010-09-22 10:31:52 +01002116 while (!list_empty(&ring->request_list)) {
2117 struct drm_i915_gem_request *request;
Chris Wilson9375e442010-09-19 12:21:28 +01002118
Chris Wilsondfaae392010-09-22 10:31:52 +01002119 request = list_first_entry(&ring->request_list,
2120 struct drm_i915_gem_request,
2121 list);
2122
Mika Kuoppala0e50e962013-05-02 16:48:08 +03002123 i915_gem_free_request(request);
Chris Wilsondfaae392010-09-22 10:31:52 +01002124 }
2125
2126 while (!list_empty(&ring->active_list)) {
Chris Wilson05394f32010-11-08 19:18:58 +00002127 struct drm_i915_gem_object *obj;
Eric Anholt673a3942008-07-30 12:06:12 -07002128
Chris Wilson05394f32010-11-08 19:18:58 +00002129 obj = list_first_entry(&ring->active_list,
2130 struct drm_i915_gem_object,
2131 ring_list);
Eric Anholt673a3942008-07-30 12:06:12 -07002132
Chris Wilson05394f32010-11-08 19:18:58 +00002133 i915_gem_object_move_to_inactive(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002134 }
Eric Anholt673a3942008-07-30 12:06:12 -07002135}
2136
Chris Wilson312817a2010-11-22 11:50:11 +00002137static void i915_gem_reset_fences(struct drm_device *dev)
2138{
2139 struct drm_i915_private *dev_priv = dev->dev_private;
2140 int i;
2141
Daniel Vetter4b9de732011-10-09 21:52:02 +02002142 for (i = 0; i < dev_priv->num_fence_regs; i++) {
Chris Wilson312817a2010-11-22 11:50:11 +00002143 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
Chris Wilson7d2cb392010-11-27 17:38:29 +00002144
Chris Wilsonada726c2012-04-17 15:31:32 +01002145 if (reg->obj)
2146 i915_gem_object_fence_lost(reg->obj);
Chris Wilson7d2cb392010-11-27 17:38:29 +00002147
Chris Wilsonf9c513e2013-03-26 11:29:27 +00002148 i915_gem_write_fence(dev, i, NULL);
2149
Chris Wilsonada726c2012-04-17 15:31:32 +01002150 reg->pin_count = 0;
2151 reg->obj = NULL;
2152 INIT_LIST_HEAD(&reg->lru_list);
Chris Wilson312817a2010-11-22 11:50:11 +00002153 }
Chris Wilsonada726c2012-04-17 15:31:32 +01002154
2155 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
Chris Wilson312817a2010-11-22 11:50:11 +00002156}
2157
Chris Wilson069efc12010-09-30 16:53:18 +01002158void i915_gem_reset(struct drm_device *dev)
Eric Anholt673a3942008-07-30 12:06:12 -07002159{
Chris Wilsondfaae392010-09-22 10:31:52 +01002160 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson05394f32010-11-08 19:18:58 +00002161 struct drm_i915_gem_object *obj;
Chris Wilsonb4519512012-05-11 14:29:30 +01002162 struct intel_ring_buffer *ring;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002163 int i;
Eric Anholt673a3942008-07-30 12:06:12 -07002164
Chris Wilsonb4519512012-05-11 14:29:30 +01002165 for_each_ring(ring, dev_priv, i)
2166 i915_gem_reset_ring_lists(dev_priv, ring);
Chris Wilsondfaae392010-09-22 10:31:52 +01002167
Chris Wilsondfaae392010-09-22 10:31:52 +01002168 /* Move everything out of the GPU domains to ensure we do any
2169 * necessary invalidation upon reuse.
2170 */
Chris Wilson05394f32010-11-08 19:18:58 +00002171 list_for_each_entry(obj,
Chris Wilson77f01232010-09-19 12:31:36 +01002172 &dev_priv->mm.inactive_list,
Chris Wilson69dc4982010-10-19 10:36:51 +01002173 mm_list)
Chris Wilson77f01232010-09-19 12:31:36 +01002174 {
Chris Wilson05394f32010-11-08 19:18:58 +00002175 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
Chris Wilson77f01232010-09-19 12:31:36 +01002176 }
Chris Wilson069efc12010-09-30 16:53:18 +01002177
2178 /* The fence registers are invalidated so clear them out */
Chris Wilson312817a2010-11-22 11:50:11 +00002179 i915_gem_reset_fences(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07002180}
2181
2182/**
2183 * This function clears the request list as sequence numbers are passed.
2184 */
Chris Wilsona71d8d92012-02-15 11:25:36 +00002185void
Chris Wilsondb53a302011-02-03 11:57:46 +00002186i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07002187{
Eric Anholt673a3942008-07-30 12:06:12 -07002188 uint32_t seqno;
2189
Chris Wilsondb53a302011-02-03 11:57:46 +00002190 if (list_empty(&ring->request_list))
Karsten Wiese6c0594a2009-02-23 15:07:57 +01002191 return;
2192
Chris Wilsondb53a302011-02-03 11:57:46 +00002193 WARN_ON(i915_verify_lists(ring->dev));
Eric Anholt673a3942008-07-30 12:06:12 -07002194
Chris Wilsonb2eadbc2012-08-09 10:58:30 +01002195 seqno = ring->get_seqno(ring, true);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002196
Zou Nan hai852835f2010-05-21 09:08:56 +08002197 while (!list_empty(&ring->request_list)) {
Eric Anholt673a3942008-07-30 12:06:12 -07002198 struct drm_i915_gem_request *request;
Eric Anholt673a3942008-07-30 12:06:12 -07002199
Zou Nan hai852835f2010-05-21 09:08:56 +08002200 request = list_first_entry(&ring->request_list,
Eric Anholt673a3942008-07-30 12:06:12 -07002201 struct drm_i915_gem_request,
2202 list);
Eric Anholt673a3942008-07-30 12:06:12 -07002203
Chris Wilsondfaae392010-09-22 10:31:52 +01002204 if (!i915_seqno_passed(seqno, request->seqno))
Eric Anholt673a3942008-07-30 12:06:12 -07002205 break;
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002206
Chris Wilsondb53a302011-02-03 11:57:46 +00002207 trace_i915_gem_request_retire(ring, request->seqno);
Chris Wilsona71d8d92012-02-15 11:25:36 +00002208 /* We know the GPU must have read the request to have
2209 * sent us the seqno + interrupt, so use the position
2210 * of tail of the request to update the last known position
2211 * of the GPU head.
2212 */
2213 ring->last_retired_head = request->tail;
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002214
Mika Kuoppala0e50e962013-05-02 16:48:08 +03002215 i915_gem_free_request(request);
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002216 }
2217
2218 /* Move any buffers on the active list that are no longer referenced
2219 * by the ringbuffer to the flushing/inactive lists as appropriate.
2220 */
2221 while (!list_empty(&ring->active_list)) {
Chris Wilson05394f32010-11-08 19:18:58 +00002222 struct drm_i915_gem_object *obj;
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002223
Akshay Joshi0206e352011-08-16 15:34:10 -04002224 obj = list_first_entry(&ring->active_list,
Chris Wilson05394f32010-11-08 19:18:58 +00002225 struct drm_i915_gem_object,
2226 ring_list);
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002227
Chris Wilson0201f1e2012-07-20 12:41:01 +01002228 if (!i915_seqno_passed(seqno, obj->last_read_seqno))
Chris Wilsonb84d5f02010-09-18 01:38:04 +01002229 break;
2230
Chris Wilson65ce3022012-07-20 12:41:02 +01002231 i915_gem_object_move_to_inactive(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002232 }
Chris Wilson9d34e5d2009-09-24 05:26:06 +01002233
Chris Wilsondb53a302011-02-03 11:57:46 +00002234 if (unlikely(ring->trace_irq_seqno &&
2235 i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002236 ring->irq_put(ring);
Chris Wilsondb53a302011-02-03 11:57:46 +00002237 ring->trace_irq_seqno = 0;
Chris Wilson9d34e5d2009-09-24 05:26:06 +01002238 }
Chris Wilson23bc5982010-09-29 16:10:57 +01002239
Chris Wilsondb53a302011-02-03 11:57:46 +00002240 WARN_ON(i915_verify_lists(ring->dev));
Eric Anholt673a3942008-07-30 12:06:12 -07002241}
2242
2243void
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01002244i915_gem_retire_requests(struct drm_device *dev)
2245{
2246 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsonb4519512012-05-11 14:29:30 +01002247 struct intel_ring_buffer *ring;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002248 int i;
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01002249
Chris Wilsonb4519512012-05-11 14:29:30 +01002250 for_each_ring(ring, dev_priv, i)
2251 i915_gem_retire_requests_ring(ring);
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01002252}
2253
Daniel Vetter75ef9da2010-08-21 00:25:16 +02002254static void
Eric Anholt673a3942008-07-30 12:06:12 -07002255i915_gem_retire_work_handler(struct work_struct *work)
2256{
2257 drm_i915_private_t *dev_priv;
2258 struct drm_device *dev;
Chris Wilsonb4519512012-05-11 14:29:30 +01002259 struct intel_ring_buffer *ring;
Chris Wilson0a587052011-01-09 21:05:44 +00002260 bool idle;
2261 int i;
Eric Anholt673a3942008-07-30 12:06:12 -07002262
2263 dev_priv = container_of(work, drm_i915_private_t,
2264 mm.retire_work.work);
2265 dev = dev_priv->dev;
2266
Chris Wilson891b48c2010-09-29 12:26:37 +01002267 /* Come back later if the device is busy... */
2268 if (!mutex_trylock(&dev->struct_mutex)) {
Chris Wilsonbcb45082012-10-05 17:02:57 +01002269 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2270 round_jiffies_up_relative(HZ));
Chris Wilson891b48c2010-09-29 12:26:37 +01002271 return;
2272 }
2273
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01002274 i915_gem_retire_requests(dev);
Zou Nan haid1b851f2010-05-21 09:08:57 +08002275
Chris Wilson0a587052011-01-09 21:05:44 +00002276 /* Send a periodic flush down the ring so we don't hold onto GEM
2277 * objects indefinitely.
2278 */
2279 idle = true;
Chris Wilsonb4519512012-05-11 14:29:30 +01002280 for_each_ring(ring, dev_priv, i) {
Chris Wilson3bb73ab2012-07-20 12:40:59 +01002281 if (ring->gpu_caches_dirty)
2282 i915_add_request(ring, NULL, NULL);
Chris Wilson0a587052011-01-09 21:05:44 +00002283
2284 idle &= list_empty(&ring->request_list);
2285 }
2286
2287 if (!dev_priv->mm.suspended && !idle)
Chris Wilsonbcb45082012-10-05 17:02:57 +01002288 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
2289 round_jiffies_up_relative(HZ));
Chris Wilsonf047e392012-07-21 12:31:41 +01002290 if (idle)
2291 intel_mark_idle(dev);
Chris Wilson0a587052011-01-09 21:05:44 +00002292
Eric Anholt673a3942008-07-30 12:06:12 -07002293 mutex_unlock(&dev->struct_mutex);
2294}
2295
Ben Widawsky5816d642012-04-11 11:18:19 -07002296/**
Daniel Vetter30dfebf2012-06-01 15:21:23 +02002297 * Ensures that an object will eventually get non-busy by flushing any required
2298 * write domains, emitting any outstanding lazy request and retiring and
2299 * completed requests.
2300 */
2301static int
2302i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
2303{
2304 int ret;
2305
2306 if (obj->active) {
Chris Wilson0201f1e2012-07-20 12:41:01 +01002307 ret = i915_gem_check_olr(obj->ring, obj->last_read_seqno);
Daniel Vetter30dfebf2012-06-01 15:21:23 +02002308 if (ret)
2309 return ret;
2310
Daniel Vetter30dfebf2012-06-01 15:21:23 +02002311 i915_gem_retire_requests_ring(obj->ring);
2312 }
2313
2314 return 0;
2315}
2316
2317/**
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002318 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
2319 * @DRM_IOCTL_ARGS: standard ioctl arguments
2320 *
2321 * Returns 0 if successful, else an error is returned with the remaining time in
2322 * the timeout parameter.
2323 * -ETIME: object is still busy after timeout
2324 * -ERESTARTSYS: signal interrupted the wait
2325 * -ENONENT: object doesn't exist
2326 * Also possible, but rare:
2327 * -EAGAIN: GPU wedged
2328 * -ENOMEM: damn
2329 * -ENODEV: Internal IRQ fail
2330 * -E?: The add request failed
2331 *
2332 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
2333 * non-zero timeout parameter the wait ioctl will wait for the given number of
2334 * nanoseconds on an object becoming unbusy. Since the wait itself does so
2335 * without holding struct_mutex the object may become re-busied before this
2336 * function completes. A similar but shorter * race condition exists in the busy
2337 * ioctl
2338 */
2339int
2340i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
2341{
Daniel Vetterf69061b2012-12-06 09:01:42 +01002342 drm_i915_private_t *dev_priv = dev->dev_private;
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002343 struct drm_i915_gem_wait *args = data;
2344 struct drm_i915_gem_object *obj;
2345 struct intel_ring_buffer *ring = NULL;
Ben Widawskyeac1f142012-06-05 15:24:24 -07002346 struct timespec timeout_stack, *timeout = NULL;
Daniel Vetterf69061b2012-12-06 09:01:42 +01002347 unsigned reset_counter;
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002348 u32 seqno = 0;
2349 int ret = 0;
2350
Ben Widawskyeac1f142012-06-05 15:24:24 -07002351 if (args->timeout_ns >= 0) {
2352 timeout_stack = ns_to_timespec(args->timeout_ns);
2353 timeout = &timeout_stack;
2354 }
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002355
2356 ret = i915_mutex_lock_interruptible(dev);
2357 if (ret)
2358 return ret;
2359
2360 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
2361 if (&obj->base == NULL) {
2362 mutex_unlock(&dev->struct_mutex);
2363 return -ENOENT;
2364 }
2365
Daniel Vetter30dfebf2012-06-01 15:21:23 +02002366 /* Need to make sure the object gets inactive eventually. */
2367 ret = i915_gem_object_flush_active(obj);
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002368 if (ret)
2369 goto out;
2370
2371 if (obj->active) {
Chris Wilson0201f1e2012-07-20 12:41:01 +01002372 seqno = obj->last_read_seqno;
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002373 ring = obj->ring;
2374 }
2375
2376 if (seqno == 0)
2377 goto out;
2378
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002379 /* Do this after OLR check to make sure we make forward progress polling
2380 * on this IOCTL with a 0 timeout (like busy ioctl)
2381 */
2382 if (!args->timeout_ns) {
2383 ret = -ETIME;
2384 goto out;
2385 }
2386
2387 drm_gem_object_unreference(&obj->base);
Daniel Vetterf69061b2012-12-06 09:01:42 +01002388 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002389 mutex_unlock(&dev->struct_mutex);
2390
Daniel Vetterf69061b2012-12-06 09:01:42 +01002391 ret = __wait_seqno(ring, seqno, reset_counter, true, timeout);
Chris Wilson4f42f4e2013-04-26 16:22:46 +03002392 if (timeout)
Ben Widawskyeac1f142012-06-05 15:24:24 -07002393 args->timeout_ns = timespec_to_ns(timeout);
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07002394 return ret;
2395
2396out:
2397 drm_gem_object_unreference(&obj->base);
2398 mutex_unlock(&dev->struct_mutex);
2399 return ret;
2400}
2401
2402/**
Ben Widawsky5816d642012-04-11 11:18:19 -07002403 * i915_gem_object_sync - sync an object to a ring.
2404 *
2405 * @obj: object which may be in use on another ring.
2406 * @to: ring we wish to use the object on. May be NULL.
2407 *
2408 * This code is meant to abstract object synchronization with the GPU.
2409 * Calling with NULL implies synchronizing the object with the CPU
2410 * rather than a particular GPU ring.
2411 *
2412 * Returns 0 if successful, else propagates up the lower layer error.
2413 */
Ben Widawsky2911a352012-04-05 14:47:36 -07002414int
2415i915_gem_object_sync(struct drm_i915_gem_object *obj,
2416 struct intel_ring_buffer *to)
2417{
2418 struct intel_ring_buffer *from = obj->ring;
2419 u32 seqno;
2420 int ret, idx;
2421
2422 if (from == NULL || to == from)
2423 return 0;
2424
Ben Widawsky5816d642012-04-11 11:18:19 -07002425 if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
Chris Wilson0201f1e2012-07-20 12:41:01 +01002426 return i915_gem_object_wait_rendering(obj, false);
Ben Widawsky2911a352012-04-05 14:47:36 -07002427
2428 idx = intel_ring_sync_index(from, to);
2429
Chris Wilson0201f1e2012-07-20 12:41:01 +01002430 seqno = obj->last_read_seqno;
Ben Widawsky2911a352012-04-05 14:47:36 -07002431 if (seqno <= from->sync_seqno[idx])
2432 return 0;
2433
Ben Widawskyb4aca012012-04-25 20:50:12 -07002434 ret = i915_gem_check_olr(obj->ring, seqno);
2435 if (ret)
2436 return ret;
Ben Widawsky2911a352012-04-05 14:47:36 -07002437
Ben Widawsky1500f7e2012-04-11 11:18:21 -07002438 ret = to->sync_to(to, from, seqno);
Ben Widawskye3a5a222012-04-11 11:18:20 -07002439 if (!ret)
Mika Kuoppala7b01e262012-11-28 17:18:45 +02002440 /* We use last_read_seqno because sync_to()
2441 * might have just caused seqno wrap under
2442 * the radar.
2443 */
2444 from->sync_seqno[idx] = obj->last_read_seqno;
Ben Widawsky2911a352012-04-05 14:47:36 -07002445
Ben Widawskye3a5a222012-04-11 11:18:20 -07002446 return ret;
Ben Widawsky2911a352012-04-05 14:47:36 -07002447}
2448
Chris Wilsonb5ffc9b2011-04-13 22:06:03 +01002449static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2450{
2451 u32 old_write_domain, old_read_domains;
2452
Chris Wilsonb5ffc9b2011-04-13 22:06:03 +01002453 /* Force a pagefault for domain tracking on next user access */
2454 i915_gem_release_mmap(obj);
2455
Keith Packardb97c3d92011-06-24 21:02:59 -07002456 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2457 return;
2458
Chris Wilson97c809fd2012-10-09 19:24:38 +01002459 /* Wait for any direct GTT access to complete */
2460 mb();
2461
Chris Wilsonb5ffc9b2011-04-13 22:06:03 +01002462 old_read_domains = obj->base.read_domains;
2463 old_write_domain = obj->base.write_domain;
2464
2465 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2466 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2467
2468 trace_i915_gem_object_change_domain(obj,
2469 old_read_domains,
2470 old_write_domain);
2471}
2472
Eric Anholt673a3942008-07-30 12:06:12 -07002473/**
2474 * Unbinds an object from the GTT aperture.
2475 */
Jesse Barnes0f973f22009-01-26 17:10:45 -08002476int
Chris Wilson05394f32010-11-08 19:18:58 +00002477i915_gem_object_unbind(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07002478{
Daniel Vetter7bddb012012-02-09 17:15:47 +01002479 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
Chris Wilson43e28f02013-01-08 10:53:09 +00002480 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002481
Chris Wilson05394f32010-11-08 19:18:58 +00002482 if (obj->gtt_space == NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07002483 return 0;
2484
Chris Wilson31d8d652012-05-24 19:11:20 +01002485 if (obj->pin_count)
2486 return -EBUSY;
Eric Anholt673a3942008-07-30 12:06:12 -07002487
Chris Wilsonc4670ad2012-08-20 10:23:27 +01002488 BUG_ON(obj->pages == NULL);
2489
Chris Wilsona8198ee2011-04-13 22:04:09 +01002490 ret = i915_gem_object_finish_gpu(obj);
Chris Wilson1488fc02012-04-24 15:47:31 +01002491 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07002492 return ret;
Chris Wilson8dc17752010-07-23 23:18:51 +01002493 /* Continue on if we fail due to EIO, the GPU is hung so we
2494 * should be safe and we need to cleanup or else we might
2495 * cause memory corruption through use-after-free.
2496 */
Chris Wilsona8198ee2011-04-13 22:04:09 +01002497
Chris Wilsonb5ffc9b2011-04-13 22:06:03 +01002498 i915_gem_object_finish_gtt(obj);
Chris Wilsona8198ee2011-04-13 22:04:09 +01002499
Daniel Vetter96b47b62009-12-15 17:50:00 +01002500 /* release the fence reg _after_ flushing */
Chris Wilsond9e86c02010-11-10 16:40:20 +00002501 ret = i915_gem_object_put_fence(obj);
Chris Wilson1488fc02012-04-24 15:47:31 +01002502 if (ret)
Chris Wilsond9e86c02010-11-10 16:40:20 +00002503 return ret;
Daniel Vetter96b47b62009-12-15 17:50:00 +01002504
Chris Wilsondb53a302011-02-03 11:57:46 +00002505 trace_i915_gem_object_unbind(obj);
2506
Daniel Vetter74898d72012-02-15 23:50:22 +01002507 if (obj->has_global_gtt_mapping)
2508 i915_gem_gtt_unbind_object(obj);
Daniel Vetter7bddb012012-02-09 17:15:47 +01002509 if (obj->has_aliasing_ppgtt_mapping) {
2510 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
2511 obj->has_aliasing_ppgtt_mapping = 0;
2512 }
Daniel Vetter74163902012-02-15 23:50:21 +01002513 i915_gem_gtt_finish_object(obj);
Ben Widawsky401c29f2013-05-31 11:28:47 -07002514 i915_gem_object_unpin_pages(obj);
Daniel Vetter7bddb012012-02-09 17:15:47 +01002515
Chris Wilson6c085a72012-08-20 11:40:46 +02002516 list_del(&obj->mm_list);
2517 list_move_tail(&obj->gtt_list, &dev_priv->mm.unbound_list);
Daniel Vetter75e9e912010-11-04 17:11:09 +01002518 /* Avoid an unnecessary call to unbind on rebind. */
Chris Wilson05394f32010-11-08 19:18:58 +00002519 obj->map_and_fenceable = true;
Eric Anholt673a3942008-07-30 12:06:12 -07002520
Chris Wilson05394f32010-11-08 19:18:58 +00002521 drm_mm_put_block(obj->gtt_space);
2522 obj->gtt_space = NULL;
2523 obj->gtt_offset = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07002524
Chris Wilson88241782011-01-07 17:09:48 +00002525 return 0;
Chris Wilson54cf91d2010-11-25 18:00:26 +00002526}
2527
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07002528int i915_gpu_idle(struct drm_device *dev)
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002529{
2530 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsonb4519512012-05-11 14:29:30 +01002531 struct intel_ring_buffer *ring;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002532 int ret, i;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002533
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002534 /* Flush everything onto the inactive list. */
Chris Wilsonb4519512012-05-11 14:29:30 +01002535 for_each_ring(ring, dev_priv, i) {
Ben Widawskyb6c74882012-08-14 14:35:14 -07002536 ret = i915_switch_context(ring, NULL, DEFAULT_CONTEXT_ID);
2537 if (ret)
2538 return ret;
2539
Chris Wilson3e960502012-11-27 16:22:54 +00002540 ret = intel_ring_idle(ring);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002541 if (ret)
2542 return ret;
2543 }
Zou Nan haid1b851f2010-05-21 09:08:57 +08002544
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01002545 return 0;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002546}
2547
Chris Wilson9ce079e2012-04-17 15:31:30 +01002548static void i965_write_fence_reg(struct drm_device *dev, int reg,
2549 struct drm_i915_gem_object *obj)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002550{
Jesse Barnesde151cf2008-11-12 10:03:55 -08002551 drm_i915_private_t *dev_priv = dev->dev_private;
Imre Deak56c844e2013-01-07 21:47:34 +02002552 int fence_reg;
2553 int fence_pitch_shift;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002554 uint64_t val;
2555
Imre Deak56c844e2013-01-07 21:47:34 +02002556 if (INTEL_INFO(dev)->gen >= 6) {
2557 fence_reg = FENCE_REG_SANDYBRIDGE_0;
2558 fence_pitch_shift = SANDYBRIDGE_FENCE_PITCH_SHIFT;
2559 } else {
2560 fence_reg = FENCE_REG_965_0;
2561 fence_pitch_shift = I965_FENCE_PITCH_SHIFT;
2562 }
2563
Chris Wilson9ce079e2012-04-17 15:31:30 +01002564 if (obj) {
2565 u32 size = obj->gtt_space->size;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002566
Chris Wilson9ce079e2012-04-17 15:31:30 +01002567 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2568 0xfffff000) << 32;
2569 val |= obj->gtt_offset & 0xfffff000;
Imre Deak56c844e2013-01-07 21:47:34 +02002570 val |= (uint64_t)((obj->stride / 128) - 1) << fence_pitch_shift;
Chris Wilson9ce079e2012-04-17 15:31:30 +01002571 if (obj->tiling_mode == I915_TILING_Y)
2572 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2573 val |= I965_FENCE_REG_VALID;
2574 } else
2575 val = 0;
Daniel Vetterc6642782010-11-12 13:46:18 +00002576
Imre Deak56c844e2013-01-07 21:47:34 +02002577 fence_reg += reg * 8;
2578 I915_WRITE64(fence_reg, val);
2579 POSTING_READ(fence_reg);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002580}
2581
Chris Wilson9ce079e2012-04-17 15:31:30 +01002582static void i915_write_fence_reg(struct drm_device *dev, int reg,
2583 struct drm_i915_gem_object *obj)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002584{
Jesse Barnesde151cf2008-11-12 10:03:55 -08002585 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson9ce079e2012-04-17 15:31:30 +01002586 u32 val;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002587
Chris Wilson9ce079e2012-04-17 15:31:30 +01002588 if (obj) {
2589 u32 size = obj->gtt_space->size;
2590 int pitch_val;
2591 int tile_width;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002592
Chris Wilson9ce079e2012-04-17 15:31:30 +01002593 WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2594 (size & -size) != size ||
2595 (obj->gtt_offset & (size - 1)),
2596 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2597 obj->gtt_offset, obj->map_and_fenceable, size);
2598
2599 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2600 tile_width = 128;
2601 else
2602 tile_width = 512;
2603
2604 /* Note: pitch better be a power of two tile widths */
2605 pitch_val = obj->stride / tile_width;
2606 pitch_val = ffs(pitch_val) - 1;
2607
2608 val = obj->gtt_offset;
2609 if (obj->tiling_mode == I915_TILING_Y)
2610 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2611 val |= I915_FENCE_SIZE_BITS(size);
2612 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2613 val |= I830_FENCE_REG_VALID;
2614 } else
2615 val = 0;
2616
2617 if (reg < 8)
2618 reg = FENCE_REG_830_0 + reg * 4;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002619 else
Chris Wilson9ce079e2012-04-17 15:31:30 +01002620 reg = FENCE_REG_945_8 + (reg - 8) * 4;
Jesse Barnes0f973f22009-01-26 17:10:45 -08002621
Chris Wilson9ce079e2012-04-17 15:31:30 +01002622 I915_WRITE(reg, val);
2623 POSTING_READ(reg);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002624}
2625
Chris Wilson9ce079e2012-04-17 15:31:30 +01002626static void i830_write_fence_reg(struct drm_device *dev, int reg,
2627 struct drm_i915_gem_object *obj)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002628{
Jesse Barnesde151cf2008-11-12 10:03:55 -08002629 drm_i915_private_t *dev_priv = dev->dev_private;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002630 uint32_t val;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002631
Chris Wilson9ce079e2012-04-17 15:31:30 +01002632 if (obj) {
2633 u32 size = obj->gtt_space->size;
2634 uint32_t pitch_val;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002635
Chris Wilson9ce079e2012-04-17 15:31:30 +01002636 WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2637 (size & -size) != size ||
2638 (obj->gtt_offset & (size - 1)),
2639 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2640 obj->gtt_offset, size);
Eric Anholte76a16d2009-05-26 17:44:56 -07002641
Chris Wilson9ce079e2012-04-17 15:31:30 +01002642 pitch_val = obj->stride / 128;
2643 pitch_val = ffs(pitch_val) - 1;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002644
Chris Wilson9ce079e2012-04-17 15:31:30 +01002645 val = obj->gtt_offset;
2646 if (obj->tiling_mode == I915_TILING_Y)
2647 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2648 val |= I830_FENCE_SIZE_BITS(size);
2649 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2650 val |= I830_FENCE_REG_VALID;
2651 } else
2652 val = 0;
Daniel Vetterc6642782010-11-12 13:46:18 +00002653
Chris Wilson9ce079e2012-04-17 15:31:30 +01002654 I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
2655 POSTING_READ(FENCE_REG_830_0 + reg * 4);
2656}
2657
Chris Wilsond0a57782012-10-09 19:24:37 +01002658inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
2659{
2660 return obj && obj->base.read_domains & I915_GEM_DOMAIN_GTT;
2661}
2662
Chris Wilson9ce079e2012-04-17 15:31:30 +01002663static void i915_gem_write_fence(struct drm_device *dev, int reg,
2664 struct drm_i915_gem_object *obj)
2665{
Chris Wilsond0a57782012-10-09 19:24:37 +01002666 struct drm_i915_private *dev_priv = dev->dev_private;
2667
2668 /* Ensure that all CPU reads are completed before installing a fence
2669 * and all writes before removing the fence.
2670 */
2671 if (i915_gem_object_needs_mb(dev_priv->fence_regs[reg].obj))
2672 mb();
2673
Chris Wilson9ce079e2012-04-17 15:31:30 +01002674 switch (INTEL_INFO(dev)->gen) {
2675 case 7:
Imre Deak56c844e2013-01-07 21:47:34 +02002676 case 6:
Chris Wilson9ce079e2012-04-17 15:31:30 +01002677 case 5:
2678 case 4: i965_write_fence_reg(dev, reg, obj); break;
2679 case 3: i915_write_fence_reg(dev, reg, obj); break;
2680 case 2: i830_write_fence_reg(dev, reg, obj); break;
Ben Widawsky7dbf9d62012-12-18 10:31:22 -08002681 default: BUG();
Chris Wilson9ce079e2012-04-17 15:31:30 +01002682 }
Chris Wilsond0a57782012-10-09 19:24:37 +01002683
2684 /* And similarly be paranoid that no direct access to this region
2685 * is reordered to before the fence is installed.
2686 */
2687 if (i915_gem_object_needs_mb(obj))
2688 mb();
Jesse Barnesde151cf2008-11-12 10:03:55 -08002689}
2690
Chris Wilson61050802012-04-17 15:31:31 +01002691static inline int fence_number(struct drm_i915_private *dev_priv,
2692 struct drm_i915_fence_reg *fence)
2693{
2694 return fence - dev_priv->fence_regs;
2695}
2696
Chris Wilson2dc8aae2013-05-22 17:08:06 +01002697struct write_fence {
2698 struct drm_device *dev;
2699 struct drm_i915_gem_object *obj;
2700 int fence;
2701};
2702
Chris Wilson25ff1192013-04-04 21:31:03 +01002703static void i915_gem_write_fence__ipi(void *data)
2704{
Chris Wilson2dc8aae2013-05-22 17:08:06 +01002705 struct write_fence *args = data;
2706
2707 /* Required for SNB+ with LLC */
Chris Wilson25ff1192013-04-04 21:31:03 +01002708 wbinvd();
Chris Wilson2dc8aae2013-05-22 17:08:06 +01002709
2710 /* Required for VLV */
2711 i915_gem_write_fence(args->dev, args->fence, args->obj);
Chris Wilson25ff1192013-04-04 21:31:03 +01002712}
2713
Chris Wilson61050802012-04-17 15:31:31 +01002714static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
2715 struct drm_i915_fence_reg *fence,
2716 bool enable)
2717{
Chris Wilson2dc8aae2013-05-22 17:08:06 +01002718 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2719 struct write_fence args = {
2720 .dev = obj->base.dev,
2721 .fence = fence_number(dev_priv, fence),
2722 .obj = enable ? obj : NULL,
2723 };
Chris Wilson61050802012-04-17 15:31:31 +01002724
Chris Wilson25ff1192013-04-04 21:31:03 +01002725 /* In order to fully serialize access to the fenced region and
2726 * the update to the fence register we need to take extreme
2727 * measures on SNB+. In theory, the write to the fence register
2728 * flushes all memory transactions before, and coupled with the
2729 * mb() placed around the register write we serialise all memory
2730 * operations with respect to the changes in the tiler. Yet, on
2731 * SNB+ we need to take a step further and emit an explicit wbinvd()
2732 * on each processor in order to manually flush all memory
2733 * transactions before updating the fence register.
Chris Wilson2dc8aae2013-05-22 17:08:06 +01002734 *
2735 * However, Valleyview complicates matter. There the wbinvd is
2736 * insufficient and unlike SNB/IVB requires the serialising
2737 * register write. (Note that that register write by itself is
2738 * conversely not sufficient for SNB+.) To compromise, we do both.
Chris Wilson25ff1192013-04-04 21:31:03 +01002739 */
Chris Wilson2dc8aae2013-05-22 17:08:06 +01002740 if (INTEL_INFO(args.dev)->gen >= 6)
2741 on_each_cpu(i915_gem_write_fence__ipi, &args, 1);
2742 else
2743 i915_gem_write_fence(args.dev, args.fence, args.obj);
Chris Wilson61050802012-04-17 15:31:31 +01002744
2745 if (enable) {
Chris Wilson2dc8aae2013-05-22 17:08:06 +01002746 obj->fence_reg = args.fence;
Chris Wilson61050802012-04-17 15:31:31 +01002747 fence->obj = obj;
2748 list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
2749 } else {
2750 obj->fence_reg = I915_FENCE_REG_NONE;
2751 fence->obj = NULL;
2752 list_del_init(&fence->lru_list);
2753 }
2754}
2755
Chris Wilsond9e86c02010-11-10 16:40:20 +00002756static int
Chris Wilsond0a57782012-10-09 19:24:37 +01002757i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
Chris Wilsond9e86c02010-11-10 16:40:20 +00002758{
Chris Wilson1c293ea2012-04-17 15:31:27 +01002759 if (obj->last_fenced_seqno) {
Chris Wilson86d5bc32012-07-20 12:41:04 +01002760 int ret = i915_wait_seqno(obj->ring, obj->last_fenced_seqno);
Chris Wilson18991842012-04-17 15:31:29 +01002761 if (ret)
2762 return ret;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002763
2764 obj->last_fenced_seqno = 0;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002765 }
2766
Chris Wilson86d5bc32012-07-20 12:41:04 +01002767 obj->fenced_gpu_access = false;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002768 return 0;
2769}
2770
2771int
2772i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2773{
Chris Wilson61050802012-04-17 15:31:31 +01002774 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
Chris Wilsonf9c513e2013-03-26 11:29:27 +00002775 struct drm_i915_fence_reg *fence;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002776 int ret;
2777
Chris Wilsond0a57782012-10-09 19:24:37 +01002778 ret = i915_gem_object_wait_fence(obj);
Chris Wilsond9e86c02010-11-10 16:40:20 +00002779 if (ret)
2780 return ret;
2781
Chris Wilson61050802012-04-17 15:31:31 +01002782 if (obj->fence_reg == I915_FENCE_REG_NONE)
2783 return 0;
Chris Wilson1690e1e2011-12-14 13:57:08 +01002784
Chris Wilsonf9c513e2013-03-26 11:29:27 +00002785 fence = &dev_priv->fence_regs[obj->fence_reg];
2786
Chris Wilson61050802012-04-17 15:31:31 +01002787 i915_gem_object_fence_lost(obj);
Chris Wilsonf9c513e2013-03-26 11:29:27 +00002788 i915_gem_object_update_fence(obj, fence, false);
Chris Wilsond9e86c02010-11-10 16:40:20 +00002789
2790 return 0;
2791}
2792
2793static struct drm_i915_fence_reg *
Chris Wilsona360bb12012-04-17 15:31:25 +01002794i915_find_fence_reg(struct drm_device *dev)
Daniel Vetterae3db242010-02-19 11:51:58 +01002795{
Daniel Vetterae3db242010-02-19 11:51:58 +01002796 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson8fe301a2012-04-17 15:31:28 +01002797 struct drm_i915_fence_reg *reg, *avail;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002798 int i;
Daniel Vetterae3db242010-02-19 11:51:58 +01002799
2800 /* First try to find a free reg */
Chris Wilsond9e86c02010-11-10 16:40:20 +00002801 avail = NULL;
Daniel Vetterae3db242010-02-19 11:51:58 +01002802 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2803 reg = &dev_priv->fence_regs[i];
2804 if (!reg->obj)
Chris Wilsond9e86c02010-11-10 16:40:20 +00002805 return reg;
Daniel Vetterae3db242010-02-19 11:51:58 +01002806
Chris Wilson1690e1e2011-12-14 13:57:08 +01002807 if (!reg->pin_count)
Chris Wilsond9e86c02010-11-10 16:40:20 +00002808 avail = reg;
Daniel Vetterae3db242010-02-19 11:51:58 +01002809 }
2810
Chris Wilsond9e86c02010-11-10 16:40:20 +00002811 if (avail == NULL)
2812 return NULL;
Daniel Vetterae3db242010-02-19 11:51:58 +01002813
2814 /* None available, try to steal one or wait for a user to finish */
Chris Wilsond9e86c02010-11-10 16:40:20 +00002815 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
Chris Wilson1690e1e2011-12-14 13:57:08 +01002816 if (reg->pin_count)
Daniel Vetterae3db242010-02-19 11:51:58 +01002817 continue;
2818
Chris Wilson8fe301a2012-04-17 15:31:28 +01002819 return reg;
Daniel Vetterae3db242010-02-19 11:51:58 +01002820 }
2821
Chris Wilson8fe301a2012-04-17 15:31:28 +01002822 return NULL;
Daniel Vetterae3db242010-02-19 11:51:58 +01002823}
2824
Jesse Barnesde151cf2008-11-12 10:03:55 -08002825/**
Chris Wilson9a5a53b2012-03-22 15:10:00 +00002826 * i915_gem_object_get_fence - set up fencing for an object
Jesse Barnesde151cf2008-11-12 10:03:55 -08002827 * @obj: object to map through a fence reg
2828 *
2829 * When mapping objects through the GTT, userspace wants to be able to write
2830 * to them without having to worry about swizzling if the object is tiled.
Jesse Barnesde151cf2008-11-12 10:03:55 -08002831 * This function walks the fence regs looking for a free one for @obj,
2832 * stealing one if it can't find any.
2833 *
2834 * It then sets up the reg based on the object's properties: address, pitch
2835 * and tiling format.
Chris Wilson9a5a53b2012-03-22 15:10:00 +00002836 *
2837 * For an untiled surface, this removes any existing fence.
Jesse Barnesde151cf2008-11-12 10:03:55 -08002838 */
Chris Wilson8c4b8c32009-06-17 22:08:52 +01002839int
Chris Wilson06d98132012-04-17 15:31:24 +01002840i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002841{
Chris Wilson05394f32010-11-08 19:18:58 +00002842 struct drm_device *dev = obj->base.dev;
Jesse Barnes79e53942008-11-07 14:24:08 -08002843 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson14415742012-04-17 15:31:33 +01002844 bool enable = obj->tiling_mode != I915_TILING_NONE;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002845 struct drm_i915_fence_reg *reg;
Daniel Vetterae3db242010-02-19 11:51:58 +01002846 int ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002847
Chris Wilson14415742012-04-17 15:31:33 +01002848 /* Have we updated the tiling parameters upon the object and so
2849 * will need to serialise the write to the associated fence register?
2850 */
Chris Wilson5d82e3e2012-04-21 16:23:23 +01002851 if (obj->fence_dirty) {
Chris Wilsond0a57782012-10-09 19:24:37 +01002852 ret = i915_gem_object_wait_fence(obj);
Chris Wilson14415742012-04-17 15:31:33 +01002853 if (ret)
2854 return ret;
2855 }
Chris Wilson9a5a53b2012-03-22 15:10:00 +00002856
Chris Wilsond9e86c02010-11-10 16:40:20 +00002857 /* Just update our place in the LRU if our fence is getting reused. */
Chris Wilson05394f32010-11-08 19:18:58 +00002858 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2859 reg = &dev_priv->fence_regs[obj->fence_reg];
Chris Wilson5d82e3e2012-04-21 16:23:23 +01002860 if (!obj->fence_dirty) {
Chris Wilson14415742012-04-17 15:31:33 +01002861 list_move_tail(&reg->lru_list,
2862 &dev_priv->mm.fence_list);
2863 return 0;
2864 }
2865 } else if (enable) {
2866 reg = i915_find_fence_reg(dev);
2867 if (reg == NULL)
2868 return -EDEADLK;
Chris Wilsond9e86c02010-11-10 16:40:20 +00002869
Chris Wilson14415742012-04-17 15:31:33 +01002870 if (reg->obj) {
2871 struct drm_i915_gem_object *old = reg->obj;
2872
Chris Wilsond0a57782012-10-09 19:24:37 +01002873 ret = i915_gem_object_wait_fence(old);
Chris Wilson29c5a582011-03-17 15:23:22 +00002874 if (ret)
2875 return ret;
2876
Chris Wilson14415742012-04-17 15:31:33 +01002877 i915_gem_object_fence_lost(old);
Chris Wilson29c5a582011-03-17 15:23:22 +00002878 }
Chris Wilson14415742012-04-17 15:31:33 +01002879 } else
Eric Anholta09ba7f2009-08-29 12:49:51 -07002880 return 0;
Eric Anholta09ba7f2009-08-29 12:49:51 -07002881
Chris Wilson14415742012-04-17 15:31:33 +01002882 i915_gem_object_update_fence(obj, reg, enable);
Chris Wilson5d82e3e2012-04-21 16:23:23 +01002883 obj->fence_dirty = false;
Chris Wilson14415742012-04-17 15:31:33 +01002884
Chris Wilson9ce079e2012-04-17 15:31:30 +01002885 return 0;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002886}
2887
Chris Wilson42d6ab42012-07-26 11:49:32 +01002888static bool i915_gem_valid_gtt_space(struct drm_device *dev,
2889 struct drm_mm_node *gtt_space,
2890 unsigned long cache_level)
2891{
2892 struct drm_mm_node *other;
2893
2894 /* On non-LLC machines we have to be careful when putting differing
2895 * types of snoopable memory together to avoid the prefetcher
Damien Lespiau4239ca72012-12-03 16:26:16 +00002896 * crossing memory domains and dying.
Chris Wilson42d6ab42012-07-26 11:49:32 +01002897 */
2898 if (HAS_LLC(dev))
2899 return true;
2900
2901 if (gtt_space == NULL)
2902 return true;
2903
2904 if (list_empty(&gtt_space->node_list))
2905 return true;
2906
2907 other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
2908 if (other->allocated && !other->hole_follows && other->color != cache_level)
2909 return false;
2910
2911 other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
2912 if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
2913 return false;
2914
2915 return true;
2916}
2917
2918static void i915_gem_verify_gtt(struct drm_device *dev)
2919{
2920#if WATCH_GTT
2921 struct drm_i915_private *dev_priv = dev->dev_private;
2922 struct drm_i915_gem_object *obj;
2923 int err = 0;
2924
2925 list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
2926 if (obj->gtt_space == NULL) {
2927 printk(KERN_ERR "object found on GTT list with no space reserved\n");
2928 err++;
2929 continue;
2930 }
2931
2932 if (obj->cache_level != obj->gtt_space->color) {
2933 printk(KERN_ERR "object reserved space [%08lx, %08lx] with wrong color, cache_level=%x, color=%lx\n",
2934 obj->gtt_space->start,
2935 obj->gtt_space->start + obj->gtt_space->size,
2936 obj->cache_level,
2937 obj->gtt_space->color);
2938 err++;
2939 continue;
2940 }
2941
2942 if (!i915_gem_valid_gtt_space(dev,
2943 obj->gtt_space,
2944 obj->cache_level)) {
2945 printk(KERN_ERR "invalid GTT space found at [%08lx, %08lx] - color=%x\n",
2946 obj->gtt_space->start,
2947 obj->gtt_space->start + obj->gtt_space->size,
2948 obj->cache_level);
2949 err++;
2950 continue;
2951 }
2952 }
2953
2954 WARN_ON(err);
2955#endif
2956}
2957
Jesse Barnesde151cf2008-11-12 10:03:55 -08002958/**
Eric Anholt673a3942008-07-30 12:06:12 -07002959 * Finds free space in the GTT aperture and binds the object there.
2960 */
2961static int
Chris Wilson05394f32010-11-08 19:18:58 +00002962i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
Daniel Vetter920afa72010-09-16 17:54:23 +02002963 unsigned alignment,
Chris Wilson86a1ee22012-08-11 15:41:04 +01002964 bool map_and_fenceable,
2965 bool nonblocking)
Eric Anholt673a3942008-07-30 12:06:12 -07002966{
Chris Wilson05394f32010-11-08 19:18:58 +00002967 struct drm_device *dev = obj->base.dev;
Eric Anholt673a3942008-07-30 12:06:12 -07002968 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsondc9dd7a2012-12-07 20:37:07 +00002969 struct drm_mm_node *node;
Daniel Vetter5e783302010-11-14 22:32:36 +01002970 u32 size, fence_size, fence_alignment, unfenced_alignment;
Daniel Vetter75e9e912010-11-04 17:11:09 +01002971 bool mappable, fenceable;
Ben Widawsky0a9ae0d2013-05-25 12:26:35 -07002972 size_t gtt_max = map_and_fenceable ?
2973 dev_priv->gtt.mappable_end : dev_priv->gtt.total;
Chris Wilson07f73f62009-09-14 16:50:30 +01002974 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002975
Chris Wilsone28f8712011-07-18 13:11:49 -07002976 fence_size = i915_gem_get_gtt_size(dev,
2977 obj->base.size,
2978 obj->tiling_mode);
2979 fence_alignment = i915_gem_get_gtt_alignment(dev,
2980 obj->base.size,
Imre Deakd8651102013-01-07 21:47:33 +02002981 obj->tiling_mode, true);
Chris Wilsone28f8712011-07-18 13:11:49 -07002982 unfenced_alignment =
Imre Deakd8651102013-01-07 21:47:33 +02002983 i915_gem_get_gtt_alignment(dev,
Chris Wilsone28f8712011-07-18 13:11:49 -07002984 obj->base.size,
Imre Deakd8651102013-01-07 21:47:33 +02002985 obj->tiling_mode, false);
Chris Wilsona00b10c2010-09-24 21:15:47 +01002986
Eric Anholt673a3942008-07-30 12:06:12 -07002987 if (alignment == 0)
Daniel Vetter5e783302010-11-14 22:32:36 +01002988 alignment = map_and_fenceable ? fence_alignment :
2989 unfenced_alignment;
Daniel Vetter75e9e912010-11-04 17:11:09 +01002990 if (map_and_fenceable && alignment & (fence_alignment - 1)) {
Eric Anholt673a3942008-07-30 12:06:12 -07002991 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2992 return -EINVAL;
2993 }
2994
Chris Wilson05394f32010-11-08 19:18:58 +00002995 size = map_and_fenceable ? fence_size : obj->base.size;
Chris Wilsona00b10c2010-09-24 21:15:47 +01002996
Chris Wilson654fc602010-05-27 13:18:21 +01002997 /* If the object is bigger than the entire aperture, reject it early
2998 * before evicting everything in a vain attempt to find space.
2999 */
Ben Widawsky0a9ae0d2013-05-25 12:26:35 -07003000 if (obj->base.size > gtt_max) {
Chris Wilsona36689c2013-05-21 16:58:49 +01003001 DRM_ERROR("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%ld\n",
3002 obj->base.size,
3003 map_and_fenceable ? "mappable" : "total",
Ben Widawsky0a9ae0d2013-05-25 12:26:35 -07003004 gtt_max);
Chris Wilson654fc602010-05-27 13:18:21 +01003005 return -E2BIG;
3006 }
3007
Chris Wilson37e680a2012-06-07 15:38:42 +01003008 ret = i915_gem_object_get_pages(obj);
Chris Wilson6c085a72012-08-20 11:40:46 +02003009 if (ret)
3010 return ret;
3011
Chris Wilsonfbdda6f2012-11-20 10:45:16 +00003012 i915_gem_object_pin_pages(obj);
3013
Chris Wilsondc9dd7a2012-12-07 20:37:07 +00003014 node = kzalloc(sizeof(*node), GFP_KERNEL);
3015 if (node == NULL) {
3016 i915_gem_object_unpin_pages(obj);
3017 return -ENOMEM;
3018 }
3019
Ben Widawsky0a9ae0d2013-05-25 12:26:35 -07003020search_free:
3021 ret = drm_mm_insert_node_in_range_generic(&dev_priv->mm.gtt_space, node,
3022 size, alignment,
3023 obj->cache_level, 0, gtt_max);
Chris Wilsondc9dd7a2012-12-07 20:37:07 +00003024 if (ret) {
Daniel Vetter75e9e912010-11-04 17:11:09 +01003025 ret = i915_gem_evict_something(dev, size, alignment,
Chris Wilson42d6ab42012-07-26 11:49:32 +01003026 obj->cache_level,
Chris Wilson86a1ee22012-08-11 15:41:04 +01003027 map_and_fenceable,
3028 nonblocking);
Chris Wilsondc9dd7a2012-12-07 20:37:07 +00003029 if (ret == 0)
3030 goto search_free;
Chris Wilson97311292009-09-21 00:22:34 +01003031
Chris Wilsonfbdda6f2012-11-20 10:45:16 +00003032 i915_gem_object_unpin_pages(obj);
Chris Wilsondc9dd7a2012-12-07 20:37:07 +00003033 kfree(node);
3034 return ret;
3035 }
3036 if (WARN_ON(!i915_gem_valid_gtt_space(dev, node, obj->cache_level))) {
3037 i915_gem_object_unpin_pages(obj);
3038 drm_mm_put_block(node);
Chris Wilson42d6ab42012-07-26 11:49:32 +01003039 return -EINVAL;
Eric Anholt673a3942008-07-30 12:06:12 -07003040 }
3041
Daniel Vetter74163902012-02-15 23:50:21 +01003042 ret = i915_gem_gtt_prepare_object(obj);
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01003043 if (ret) {
Chris Wilsonfbdda6f2012-11-20 10:45:16 +00003044 i915_gem_object_unpin_pages(obj);
Chris Wilsondc9dd7a2012-12-07 20:37:07 +00003045 drm_mm_put_block(node);
Chris Wilson6c085a72012-08-20 11:40:46 +02003046 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003047 }
Eric Anholt673a3942008-07-30 12:06:12 -07003048
Chris Wilson6c085a72012-08-20 11:40:46 +02003049 list_move_tail(&obj->gtt_list, &dev_priv->mm.bound_list);
Chris Wilson05394f32010-11-08 19:18:58 +00003050 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
Chris Wilsonbf1a1092010-08-07 11:01:20 +01003051
Chris Wilsondc9dd7a2012-12-07 20:37:07 +00003052 obj->gtt_space = node;
3053 obj->gtt_offset = node->start;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003054
Daniel Vetter75e9e912010-11-04 17:11:09 +01003055 fenceable =
Chris Wilsondc9dd7a2012-12-07 20:37:07 +00003056 node->size == fence_size &&
3057 (node->start & (fence_alignment - 1)) == 0;
Chris Wilsona00b10c2010-09-24 21:15:47 +01003058
Daniel Vetter75e9e912010-11-04 17:11:09 +01003059 mappable =
Ben Widawsky5d4545a2013-01-17 12:45:15 -08003060 obj->gtt_offset + obj->base.size <= dev_priv->gtt.mappable_end;
Chris Wilsona00b10c2010-09-24 21:15:47 +01003061
Chris Wilson05394f32010-11-08 19:18:58 +00003062 obj->map_and_fenceable = mappable && fenceable;
Daniel Vetter75e9e912010-11-04 17:11:09 +01003063
Chris Wilsondb53a302011-02-03 11:57:46 +00003064 trace_i915_gem_object_bind(obj, map_and_fenceable);
Chris Wilson42d6ab42012-07-26 11:49:32 +01003065 i915_gem_verify_gtt(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07003066 return 0;
3067}
3068
3069void
Chris Wilson05394f32010-11-08 19:18:58 +00003070i915_gem_clflush_object(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07003071{
Eric Anholt673a3942008-07-30 12:06:12 -07003072 /* If we don't have a page list set up, then we're not pinned
3073 * to GPU, and we can ignore the cache flush because it'll happen
3074 * again at bind time.
3075 */
Chris Wilson05394f32010-11-08 19:18:58 +00003076 if (obj->pages == NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07003077 return;
3078
Imre Deak769ce462013-02-13 21:56:05 +02003079 /*
3080 * Stolen memory is always coherent with the GPU as it is explicitly
3081 * marked as wc by the system, or the system is cache-coherent.
3082 */
3083 if (obj->stolen)
3084 return;
3085
Chris Wilson9c23f7f2011-03-29 16:59:52 -07003086 /* If the GPU is snooping the contents of the CPU cache,
3087 * we do not need to manually clear the CPU cache lines. However,
3088 * the caches are only snooped when the render cache is
3089 * flushed/invalidated. As we always have to emit invalidations
3090 * and flushes when moving into and out of the RENDER domain, correct
3091 * snooping behaviour occurs naturally as the result of our domain
3092 * tracking.
3093 */
3094 if (obj->cache_level != I915_CACHE_NONE)
3095 return;
3096
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003097 trace_i915_gem_object_clflush(obj);
Eric Anholtcfa16a02009-05-26 18:46:16 -07003098
Chris Wilson9da3da62012-06-01 15:20:22 +01003099 drm_clflush_sg(obj->pages);
Eric Anholte47c68e2008-11-14 13:35:19 -08003100}
3101
3102/** Flushes the GTT write domain for the object if it's dirty. */
3103static void
Chris Wilson05394f32010-11-08 19:18:58 +00003104i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
Eric Anholte47c68e2008-11-14 13:35:19 -08003105{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003106 uint32_t old_write_domain;
3107
Chris Wilson05394f32010-11-08 19:18:58 +00003108 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
Eric Anholte47c68e2008-11-14 13:35:19 -08003109 return;
3110
Chris Wilson63256ec2011-01-04 18:42:07 +00003111 /* No actual flushing is required for the GTT write domain. Writes
Eric Anholte47c68e2008-11-14 13:35:19 -08003112 * to it immediately go to main memory as far as we know, so there's
3113 * no chipset flush. It also doesn't land in render cache.
Chris Wilson63256ec2011-01-04 18:42:07 +00003114 *
3115 * However, we do have to enforce the order so that all writes through
3116 * the GTT land before any writes to the device, such as updates to
3117 * the GATT itself.
Eric Anholte47c68e2008-11-14 13:35:19 -08003118 */
Chris Wilson63256ec2011-01-04 18:42:07 +00003119 wmb();
3120
Chris Wilson05394f32010-11-08 19:18:58 +00003121 old_write_domain = obj->base.write_domain;
3122 obj->base.write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003123
3124 trace_i915_gem_object_change_domain(obj,
Chris Wilson05394f32010-11-08 19:18:58 +00003125 obj->base.read_domains,
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003126 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08003127}
3128
3129/** Flushes the CPU write domain for the object if it's dirty. */
3130static void
Chris Wilson05394f32010-11-08 19:18:58 +00003131i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
Eric Anholte47c68e2008-11-14 13:35:19 -08003132{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003133 uint32_t old_write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08003134
Chris Wilson05394f32010-11-08 19:18:58 +00003135 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
Eric Anholte47c68e2008-11-14 13:35:19 -08003136 return;
3137
3138 i915_gem_clflush_object(obj);
Ben Widawskye76e9ae2012-11-04 09:21:27 -08003139 i915_gem_chipset_flush(obj->base.dev);
Chris Wilson05394f32010-11-08 19:18:58 +00003140 old_write_domain = obj->base.write_domain;
3141 obj->base.write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003142
3143 trace_i915_gem_object_change_domain(obj,
Chris Wilson05394f32010-11-08 19:18:58 +00003144 obj->base.read_domains,
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003145 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08003146}
3147
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003148/**
3149 * Moves a single object to the GTT read, and possibly write domain.
3150 *
3151 * This function returns when the move is complete, including waiting on
3152 * flushes to occur.
3153 */
Jesse Barnes79e53942008-11-07 14:24:08 -08003154int
Chris Wilson20217462010-11-23 15:26:33 +00003155i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003156{
Chris Wilson8325a092012-04-24 15:52:35 +01003157 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003158 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003159 int ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003160
Eric Anholt02354392008-11-26 13:58:13 -08003161 /* Not valid to be called on unbound objects. */
Chris Wilson05394f32010-11-08 19:18:58 +00003162 if (obj->gtt_space == NULL)
Eric Anholt02354392008-11-26 13:58:13 -08003163 return -EINVAL;
3164
Chris Wilson8d7e3de2011-02-07 15:23:02 +00003165 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3166 return 0;
3167
Chris Wilson0201f1e2012-07-20 12:41:01 +01003168 ret = i915_gem_object_wait_rendering(obj, !write);
Chris Wilson88241782011-01-07 17:09:48 +00003169 if (ret)
3170 return ret;
3171
Chris Wilson72133422010-09-13 23:56:38 +01003172 i915_gem_object_flush_cpu_write_domain(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003173
Chris Wilsond0a57782012-10-09 19:24:37 +01003174 /* Serialise direct access to this object with the barriers for
3175 * coherent writes from the GPU, by effectively invalidating the
3176 * GTT domain upon first access.
3177 */
3178 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3179 mb();
3180
Chris Wilson05394f32010-11-08 19:18:58 +00003181 old_write_domain = obj->base.write_domain;
3182 old_read_domains = obj->base.read_domains;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003183
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003184 /* It should now be out of any other write domains, and we can update
3185 * the domain values for our changes.
3186 */
Chris Wilson05394f32010-11-08 19:18:58 +00003187 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3188 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08003189 if (write) {
Chris Wilson05394f32010-11-08 19:18:58 +00003190 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3191 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3192 obj->dirty = 1;
Eric Anholte47c68e2008-11-14 13:35:19 -08003193 }
3194
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003195 trace_i915_gem_object_change_domain(obj,
3196 old_read_domains,
3197 old_write_domain);
3198
Chris Wilson8325a092012-04-24 15:52:35 +01003199 /* And bump the LRU for this access */
3200 if (i915_gem_object_is_inactive(obj))
3201 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
3202
Eric Anholte47c68e2008-11-14 13:35:19 -08003203 return 0;
3204}
3205
Chris Wilsone4ffd172011-04-04 09:44:39 +01003206int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3207 enum i915_cache_level cache_level)
3208{
Daniel Vetter7bddb012012-02-09 17:15:47 +01003209 struct drm_device *dev = obj->base.dev;
3210 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsone4ffd172011-04-04 09:44:39 +01003211 int ret;
3212
3213 if (obj->cache_level == cache_level)
3214 return 0;
3215
3216 if (obj->pin_count) {
3217 DRM_DEBUG("can not change the cache level of pinned objects\n");
3218 return -EBUSY;
3219 }
3220
Chris Wilson42d6ab42012-07-26 11:49:32 +01003221 if (!i915_gem_valid_gtt_space(dev, obj->gtt_space, cache_level)) {
3222 ret = i915_gem_object_unbind(obj);
3223 if (ret)
3224 return ret;
3225 }
3226
Chris Wilsone4ffd172011-04-04 09:44:39 +01003227 if (obj->gtt_space) {
3228 ret = i915_gem_object_finish_gpu(obj);
3229 if (ret)
3230 return ret;
3231
3232 i915_gem_object_finish_gtt(obj);
3233
3234 /* Before SandyBridge, you could not use tiling or fence
3235 * registers with snooped memory, so relinquish any fences
3236 * currently pointing to our region in the aperture.
3237 */
Chris Wilson42d6ab42012-07-26 11:49:32 +01003238 if (INTEL_INFO(dev)->gen < 6) {
Chris Wilsone4ffd172011-04-04 09:44:39 +01003239 ret = i915_gem_object_put_fence(obj);
3240 if (ret)
3241 return ret;
3242 }
3243
Daniel Vetter74898d72012-02-15 23:50:22 +01003244 if (obj->has_global_gtt_mapping)
3245 i915_gem_gtt_bind_object(obj, cache_level);
Daniel Vetter7bddb012012-02-09 17:15:47 +01003246 if (obj->has_aliasing_ppgtt_mapping)
3247 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
3248 obj, cache_level);
Chris Wilson42d6ab42012-07-26 11:49:32 +01003249
3250 obj->gtt_space->color = cache_level;
Chris Wilsone4ffd172011-04-04 09:44:39 +01003251 }
3252
3253 if (cache_level == I915_CACHE_NONE) {
3254 u32 old_read_domains, old_write_domain;
3255
3256 /* If we're coming from LLC cached, then we haven't
3257 * actually been tracking whether the data is in the
3258 * CPU cache or not, since we only allow one bit set
3259 * in obj->write_domain and have been skipping the clflushes.
3260 * Just set it to the CPU cache for now.
3261 */
3262 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3263 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
3264
3265 old_read_domains = obj->base.read_domains;
3266 old_write_domain = obj->base.write_domain;
3267
3268 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3269 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3270
3271 trace_i915_gem_object_change_domain(obj,
3272 old_read_domains,
3273 old_write_domain);
3274 }
3275
3276 obj->cache_level = cache_level;
Chris Wilson42d6ab42012-07-26 11:49:32 +01003277 i915_gem_verify_gtt(dev);
Chris Wilsone4ffd172011-04-04 09:44:39 +01003278 return 0;
3279}
3280
Ben Widawsky199adf42012-09-21 17:01:20 -07003281int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3282 struct drm_file *file)
Chris Wilsone6994ae2012-07-10 10:27:08 +01003283{
Ben Widawsky199adf42012-09-21 17:01:20 -07003284 struct drm_i915_gem_caching *args = data;
Chris Wilsone6994ae2012-07-10 10:27:08 +01003285 struct drm_i915_gem_object *obj;
3286 int ret;
3287
3288 ret = i915_mutex_lock_interruptible(dev);
3289 if (ret)
3290 return ret;
3291
3292 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3293 if (&obj->base == NULL) {
3294 ret = -ENOENT;
3295 goto unlock;
3296 }
3297
Ben Widawsky199adf42012-09-21 17:01:20 -07003298 args->caching = obj->cache_level != I915_CACHE_NONE;
Chris Wilsone6994ae2012-07-10 10:27:08 +01003299
3300 drm_gem_object_unreference(&obj->base);
3301unlock:
3302 mutex_unlock(&dev->struct_mutex);
3303 return ret;
3304}
3305
Ben Widawsky199adf42012-09-21 17:01:20 -07003306int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3307 struct drm_file *file)
Chris Wilsone6994ae2012-07-10 10:27:08 +01003308{
Ben Widawsky199adf42012-09-21 17:01:20 -07003309 struct drm_i915_gem_caching *args = data;
Chris Wilsone6994ae2012-07-10 10:27:08 +01003310 struct drm_i915_gem_object *obj;
3311 enum i915_cache_level level;
3312 int ret;
3313
Ben Widawsky199adf42012-09-21 17:01:20 -07003314 switch (args->caching) {
3315 case I915_CACHING_NONE:
Chris Wilsone6994ae2012-07-10 10:27:08 +01003316 level = I915_CACHE_NONE;
3317 break;
Ben Widawsky199adf42012-09-21 17:01:20 -07003318 case I915_CACHING_CACHED:
Chris Wilsone6994ae2012-07-10 10:27:08 +01003319 level = I915_CACHE_LLC;
3320 break;
3321 default:
3322 return -EINVAL;
3323 }
3324
Ben Widawsky3bc29132012-09-26 16:15:20 -07003325 ret = i915_mutex_lock_interruptible(dev);
3326 if (ret)
3327 return ret;
3328
Chris Wilsone6994ae2012-07-10 10:27:08 +01003329 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3330 if (&obj->base == NULL) {
3331 ret = -ENOENT;
3332 goto unlock;
3333 }
3334
3335 ret = i915_gem_object_set_cache_level(obj, level);
3336
3337 drm_gem_object_unreference(&obj->base);
3338unlock:
3339 mutex_unlock(&dev->struct_mutex);
3340 return ret;
3341}
3342
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003343/*
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003344 * Prepare buffer for display plane (scanout, cursors, etc).
3345 * Can be called from an uninterruptible phase (modesetting) and allows
3346 * any flushes to be pipelined (for pageflips).
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003347 */
3348int
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003349i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3350 u32 alignment,
Chris Wilson919926a2010-11-12 13:42:53 +00003351 struct intel_ring_buffer *pipelined)
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003352{
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003353 u32 old_read_domains, old_write_domain;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003354 int ret;
3355
Chris Wilson0be73282010-12-06 14:36:27 +00003356 if (pipelined != obj->ring) {
Ben Widawsky2911a352012-04-05 14:47:36 -07003357 ret = i915_gem_object_sync(obj, pipelined);
3358 if (ret)
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003359 return ret;
3360 }
3361
Eric Anholta7ef0642011-03-29 16:59:54 -07003362 /* The display engine is not coherent with the LLC cache on gen6. As
3363 * a result, we make sure that the pinning that is about to occur is
3364 * done with uncached PTEs. This is lowest common denominator for all
3365 * chipsets.
3366 *
3367 * However for gen6+, we could do better by using the GFDT bit instead
3368 * of uncaching, which would allow us to flush all the LLC-cached data
3369 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3370 */
3371 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
3372 if (ret)
3373 return ret;
3374
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003375 /* As the user may map the buffer once pinned in the display plane
3376 * (e.g. libkms for the bootup splash), we have to ensure that we
3377 * always use map_and_fenceable for all scanout buffers.
3378 */
Chris Wilson86a1ee22012-08-11 15:41:04 +01003379 ret = i915_gem_object_pin(obj, alignment, true, false);
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003380 if (ret)
3381 return ret;
3382
Chris Wilsonb118c1e2010-05-27 13:18:14 +01003383 i915_gem_object_flush_cpu_write_domain(obj);
3384
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003385 old_write_domain = obj->base.write_domain;
Chris Wilson05394f32010-11-08 19:18:58 +00003386 old_read_domains = obj->base.read_domains;
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003387
3388 /* It should now be out of any other write domains, and we can update
3389 * the domain values for our changes.
3390 */
Chris Wilsone5f1d962012-07-20 12:41:00 +01003391 obj->base.write_domain = 0;
Chris Wilson05394f32010-11-08 19:18:58 +00003392 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003393
3394 trace_i915_gem_object_change_domain(obj,
3395 old_read_domains,
Chris Wilson2da3b9b2011-04-14 09:41:17 +01003396 old_write_domain);
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08003397
3398 return 0;
3399}
3400
Chris Wilson85345512010-11-13 09:49:11 +00003401int
Chris Wilsona8198ee2011-04-13 22:04:09 +01003402i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
Chris Wilson85345512010-11-13 09:49:11 +00003403{
Chris Wilson88241782011-01-07 17:09:48 +00003404 int ret;
3405
Chris Wilsona8198ee2011-04-13 22:04:09 +01003406 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
Chris Wilson85345512010-11-13 09:49:11 +00003407 return 0;
3408
Chris Wilson0201f1e2012-07-20 12:41:01 +01003409 ret = i915_gem_object_wait_rendering(obj, false);
Chris Wilsonc501ae72011-12-14 13:57:23 +01003410 if (ret)
3411 return ret;
3412
Chris Wilsona8198ee2011-04-13 22:04:09 +01003413 /* Ensure that we invalidate the GPU's caches and TLBs. */
3414 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
Chris Wilsonc501ae72011-12-14 13:57:23 +01003415 return 0;
Chris Wilson85345512010-11-13 09:49:11 +00003416}
3417
Eric Anholte47c68e2008-11-14 13:35:19 -08003418/**
3419 * Moves a single object to the CPU read, and possibly write domain.
3420 *
3421 * This function returns when the move is complete, including waiting on
3422 * flushes to occur.
3423 */
Chris Wilsondabdfe02012-03-26 10:10:27 +02003424int
Chris Wilson919926a2010-11-12 13:42:53 +00003425i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
Eric Anholte47c68e2008-11-14 13:35:19 -08003426{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003427 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003428 int ret;
3429
Chris Wilson8d7e3de2011-02-07 15:23:02 +00003430 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3431 return 0;
3432
Chris Wilson0201f1e2012-07-20 12:41:01 +01003433 ret = i915_gem_object_wait_rendering(obj, !write);
Chris Wilson88241782011-01-07 17:09:48 +00003434 if (ret)
3435 return ret;
3436
Eric Anholte47c68e2008-11-14 13:35:19 -08003437 i915_gem_object_flush_gtt_write_domain(obj);
3438
Chris Wilson05394f32010-11-08 19:18:58 +00003439 old_write_domain = obj->base.write_domain;
3440 old_read_domains = obj->base.read_domains;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003441
Eric Anholte47c68e2008-11-14 13:35:19 -08003442 /* Flush the CPU cache if it's still invalid. */
Chris Wilson05394f32010-11-08 19:18:58 +00003443 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
Eric Anholte47c68e2008-11-14 13:35:19 -08003444 i915_gem_clflush_object(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08003445
Chris Wilson05394f32010-11-08 19:18:58 +00003446 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
Eric Anholte47c68e2008-11-14 13:35:19 -08003447 }
3448
3449 /* It should now be out of any other write domains, and we can update
3450 * the domain values for our changes.
3451 */
Chris Wilson05394f32010-11-08 19:18:58 +00003452 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
Eric Anholte47c68e2008-11-14 13:35:19 -08003453
3454 /* If we're writing through the CPU, then the GPU read domains will
3455 * need to be invalidated at next use.
3456 */
3457 if (write) {
Chris Wilson05394f32010-11-08 19:18:58 +00003458 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3459 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
Eric Anholte47c68e2008-11-14 13:35:19 -08003460 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003461
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003462 trace_i915_gem_object_change_domain(obj,
3463 old_read_domains,
3464 old_write_domain);
3465
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003466 return 0;
3467}
3468
Eric Anholt673a3942008-07-30 12:06:12 -07003469/* Throttle our rendering by waiting until the ring has completed our requests
3470 * emitted over 20 msec ago.
3471 *
Eric Anholtb9624422009-06-03 07:27:35 +00003472 * Note that if we were to use the current jiffies each time around the loop,
3473 * we wouldn't escape the function with any frames outstanding if the time to
3474 * render a frame was over 20ms.
3475 *
Eric Anholt673a3942008-07-30 12:06:12 -07003476 * This should get us reasonable parallelism between CPU and GPU but also
3477 * relatively low latency when blocking on a particular request to finish.
3478 */
3479static int
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003480i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07003481{
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003482 struct drm_i915_private *dev_priv = dev->dev_private;
3483 struct drm_i915_file_private *file_priv = file->driver_priv;
Eric Anholtb9624422009-06-03 07:27:35 +00003484 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003485 struct drm_i915_gem_request *request;
3486 struct intel_ring_buffer *ring = NULL;
Daniel Vetterf69061b2012-12-06 09:01:42 +01003487 unsigned reset_counter;
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003488 u32 seqno = 0;
3489 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003490
Daniel Vetter308887a2012-11-14 17:14:06 +01003491 ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
3492 if (ret)
3493 return ret;
3494
3495 ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
3496 if (ret)
3497 return ret;
Chris Wilsone110e8d2011-01-26 15:39:14 +00003498
Chris Wilson1c255952010-09-26 11:03:27 +01003499 spin_lock(&file_priv->mm.lock);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003500 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
Eric Anholtb9624422009-06-03 07:27:35 +00003501 if (time_after_eq(request->emitted_jiffies, recent_enough))
3502 break;
3503
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003504 ring = request->ring;
3505 seqno = request->seqno;
Eric Anholtb9624422009-06-03 07:27:35 +00003506 }
Daniel Vetterf69061b2012-12-06 09:01:42 +01003507 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
Chris Wilson1c255952010-09-26 11:03:27 +01003508 spin_unlock(&file_priv->mm.lock);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003509
3510 if (seqno == 0)
3511 return 0;
3512
Daniel Vetterf69061b2012-12-06 09:01:42 +01003513 ret = __wait_seqno(ring, seqno, reset_counter, true, NULL);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01003514 if (ret == 0)
3515 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
Eric Anholtb9624422009-06-03 07:27:35 +00003516
Eric Anholt673a3942008-07-30 12:06:12 -07003517 return ret;
3518}
3519
Eric Anholt673a3942008-07-30 12:06:12 -07003520int
Chris Wilson05394f32010-11-08 19:18:58 +00003521i915_gem_object_pin(struct drm_i915_gem_object *obj,
3522 uint32_t alignment,
Chris Wilson86a1ee22012-08-11 15:41:04 +01003523 bool map_and_fenceable,
3524 bool nonblocking)
Eric Anholt673a3942008-07-30 12:06:12 -07003525{
Eric Anholt673a3942008-07-30 12:06:12 -07003526 int ret;
3527
Chris Wilson7e81a422012-09-15 09:41:57 +01003528 if (WARN_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
3529 return -EBUSY;
Chris Wilsonac0c6b52010-05-27 13:18:18 +01003530
Chris Wilson05394f32010-11-08 19:18:58 +00003531 if (obj->gtt_space != NULL) {
3532 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3533 (map_and_fenceable && !obj->map_and_fenceable)) {
3534 WARN(obj->pin_count,
Chris Wilsonae7d49d2010-08-04 12:37:41 +01003535 "bo is already pinned with incorrect alignment:"
Daniel Vetter75e9e912010-11-04 17:11:09 +01003536 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3537 " obj->map_and_fenceable=%d\n",
Chris Wilson05394f32010-11-08 19:18:58 +00003538 obj->gtt_offset, alignment,
Daniel Vetter75e9e912010-11-04 17:11:09 +01003539 map_and_fenceable,
Chris Wilson05394f32010-11-08 19:18:58 +00003540 obj->map_and_fenceable);
Chris Wilsonac0c6b52010-05-27 13:18:18 +01003541 ret = i915_gem_object_unbind(obj);
3542 if (ret)
3543 return ret;
3544 }
3545 }
3546
Chris Wilson05394f32010-11-08 19:18:58 +00003547 if (obj->gtt_space == NULL) {
Chris Wilson87422672012-11-21 13:04:03 +00003548 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3549
Chris Wilsona00b10c2010-09-24 21:15:47 +01003550 ret = i915_gem_object_bind_to_gtt(obj, alignment,
Chris Wilson86a1ee22012-08-11 15:41:04 +01003551 map_and_fenceable,
3552 nonblocking);
Chris Wilson97311292009-09-21 00:22:34 +01003553 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07003554 return ret;
Chris Wilson87422672012-11-21 13:04:03 +00003555
3556 if (!dev_priv->mm.aliasing_ppgtt)
3557 i915_gem_gtt_bind_object(obj, obj->cache_level);
Chris Wilson22c344e2009-02-11 14:26:45 +00003558 }
Jesse Barnes76446ca2009-12-17 22:05:42 -05003559
Daniel Vetter74898d72012-02-15 23:50:22 +01003560 if (!obj->has_global_gtt_mapping && map_and_fenceable)
3561 i915_gem_gtt_bind_object(obj, obj->cache_level);
3562
Chris Wilson1b502472012-04-24 15:47:30 +01003563 obj->pin_count++;
Chris Wilson6299f992010-11-24 12:23:44 +00003564 obj->pin_mappable |= map_and_fenceable;
Eric Anholt673a3942008-07-30 12:06:12 -07003565
3566 return 0;
3567}
3568
3569void
Chris Wilson05394f32010-11-08 19:18:58 +00003570i915_gem_object_unpin(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07003571{
Chris Wilson05394f32010-11-08 19:18:58 +00003572 BUG_ON(obj->pin_count == 0);
3573 BUG_ON(obj->gtt_space == NULL);
Eric Anholt673a3942008-07-30 12:06:12 -07003574
Chris Wilson1b502472012-04-24 15:47:30 +01003575 if (--obj->pin_count == 0)
Chris Wilson6299f992010-11-24 12:23:44 +00003576 obj->pin_mappable = false;
Eric Anholt673a3942008-07-30 12:06:12 -07003577}
3578
3579int
3580i915_gem_pin_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00003581 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07003582{
3583 struct drm_i915_gem_pin *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00003584 struct drm_i915_gem_object *obj;
Eric Anholt673a3942008-07-30 12:06:12 -07003585 int ret;
3586
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003587 ret = i915_mutex_lock_interruptible(dev);
3588 if (ret)
3589 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003590
Chris Wilson05394f32010-11-08 19:18:58 +00003591 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00003592 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003593 ret = -ENOENT;
3594 goto unlock;
Eric Anholt673a3942008-07-30 12:06:12 -07003595 }
Eric Anholt673a3942008-07-30 12:06:12 -07003596
Chris Wilson05394f32010-11-08 19:18:58 +00003597 if (obj->madv != I915_MADV_WILLNEED) {
Chris Wilsonbb6baf72009-09-22 14:24:13 +01003598 DRM_ERROR("Attempting to pin a purgeable buffer\n");
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003599 ret = -EINVAL;
3600 goto out;
Chris Wilson3ef94da2009-09-14 16:50:29 +01003601 }
3602
Chris Wilson05394f32010-11-08 19:18:58 +00003603 if (obj->pin_filp != NULL && obj->pin_filp != file) {
Jesse Barnes79e53942008-11-07 14:24:08 -08003604 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3605 args->handle);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003606 ret = -EINVAL;
3607 goto out;
Jesse Barnes79e53942008-11-07 14:24:08 -08003608 }
3609
Chris Wilson93be8782013-01-02 10:31:22 +00003610 if (obj->user_pin_count == 0) {
Chris Wilson86a1ee22012-08-11 15:41:04 +01003611 ret = i915_gem_object_pin(obj, args->alignment, true, false);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003612 if (ret)
3613 goto out;
Eric Anholt673a3942008-07-30 12:06:12 -07003614 }
3615
Chris Wilson93be8782013-01-02 10:31:22 +00003616 obj->user_pin_count++;
3617 obj->pin_filp = file;
3618
Eric Anholt673a3942008-07-30 12:06:12 -07003619 /* XXX - flush the CPU caches for pinned objects
3620 * as the X server doesn't manage domains yet
3621 */
Eric Anholte47c68e2008-11-14 13:35:19 -08003622 i915_gem_object_flush_cpu_write_domain(obj);
Chris Wilson05394f32010-11-08 19:18:58 +00003623 args->offset = obj->gtt_offset;
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003624out:
Chris Wilson05394f32010-11-08 19:18:58 +00003625 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003626unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07003627 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003628 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003629}
3630
3631int
3632i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00003633 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07003634{
3635 struct drm_i915_gem_pin *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00003636 struct drm_i915_gem_object *obj;
Chris Wilson76c1dec2010-09-25 11:22:51 +01003637 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003638
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003639 ret = i915_mutex_lock_interruptible(dev);
3640 if (ret)
3641 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003642
Chris Wilson05394f32010-11-08 19:18:58 +00003643 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00003644 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003645 ret = -ENOENT;
3646 goto unlock;
Eric Anholt673a3942008-07-30 12:06:12 -07003647 }
Chris Wilson76c1dec2010-09-25 11:22:51 +01003648
Chris Wilson05394f32010-11-08 19:18:58 +00003649 if (obj->pin_filp != file) {
Jesse Barnes79e53942008-11-07 14:24:08 -08003650 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3651 args->handle);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003652 ret = -EINVAL;
3653 goto out;
Jesse Barnes79e53942008-11-07 14:24:08 -08003654 }
Chris Wilson05394f32010-11-08 19:18:58 +00003655 obj->user_pin_count--;
3656 if (obj->user_pin_count == 0) {
3657 obj->pin_filp = NULL;
Jesse Barnes79e53942008-11-07 14:24:08 -08003658 i915_gem_object_unpin(obj);
3659 }
Eric Anholt673a3942008-07-30 12:06:12 -07003660
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003661out:
Chris Wilson05394f32010-11-08 19:18:58 +00003662 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003663unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07003664 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003665 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003666}
3667
3668int
3669i915_gem_busy_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00003670 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07003671{
3672 struct drm_i915_gem_busy *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00003673 struct drm_i915_gem_object *obj;
Chris Wilson30dbf0c2010-09-25 10:19:17 +01003674 int ret;
3675
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003676 ret = i915_mutex_lock_interruptible(dev);
3677 if (ret)
3678 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003679
Chris Wilson05394f32010-11-08 19:18:58 +00003680 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00003681 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003682 ret = -ENOENT;
3683 goto unlock;
Eric Anholt673a3942008-07-30 12:06:12 -07003684 }
Zou Nan haid1b851f2010-05-21 09:08:57 +08003685
Chris Wilson0be555b2010-08-04 15:36:30 +01003686 /* Count all active objects as busy, even if they are currently not used
3687 * by the gpu. Users of this interface expect objects to eventually
3688 * become non-busy without any further actions, therefore emit any
3689 * necessary flushes here.
Eric Anholtc4de0a52008-12-14 19:05:04 -08003690 */
Daniel Vetter30dfebf2012-06-01 15:21:23 +02003691 ret = i915_gem_object_flush_active(obj);
3692
Chris Wilson05394f32010-11-08 19:18:58 +00003693 args->busy = obj->active;
Chris Wilsone9808ed2012-07-04 12:25:08 +01003694 if (obj->ring) {
3695 BUILD_BUG_ON(I915_NUM_RINGS > 16);
3696 args->busy |= intel_ring_flag(obj->ring) << 16;
3697 }
Eric Anholt673a3942008-07-30 12:06:12 -07003698
Chris Wilson05394f32010-11-08 19:18:58 +00003699 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003700unlock:
Eric Anholt673a3942008-07-30 12:06:12 -07003701 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003702 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003703}
3704
3705int
3706i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3707 struct drm_file *file_priv)
3708{
Akshay Joshi0206e352011-08-16 15:34:10 -04003709 return i915_gem_ring_throttle(dev, file_priv);
Eric Anholt673a3942008-07-30 12:06:12 -07003710}
3711
Chris Wilson3ef94da2009-09-14 16:50:29 +01003712int
3713i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3714 struct drm_file *file_priv)
3715{
3716 struct drm_i915_gem_madvise *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00003717 struct drm_i915_gem_object *obj;
Chris Wilson76c1dec2010-09-25 11:22:51 +01003718 int ret;
Chris Wilson3ef94da2009-09-14 16:50:29 +01003719
3720 switch (args->madv) {
3721 case I915_MADV_DONTNEED:
3722 case I915_MADV_WILLNEED:
3723 break;
3724 default:
3725 return -EINVAL;
3726 }
3727
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003728 ret = i915_mutex_lock_interruptible(dev);
3729 if (ret)
3730 return ret;
3731
Chris Wilson05394f32010-11-08 19:18:58 +00003732 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
Chris Wilsonc8725222011-02-19 11:31:06 +00003733 if (&obj->base == NULL) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003734 ret = -ENOENT;
3735 goto unlock;
Chris Wilson3ef94da2009-09-14 16:50:29 +01003736 }
Chris Wilson3ef94da2009-09-14 16:50:29 +01003737
Chris Wilson05394f32010-11-08 19:18:58 +00003738 if (obj->pin_count) {
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003739 ret = -EINVAL;
3740 goto out;
Chris Wilson3ef94da2009-09-14 16:50:29 +01003741 }
3742
Chris Wilson05394f32010-11-08 19:18:58 +00003743 if (obj->madv != __I915_MADV_PURGED)
3744 obj->madv = args->madv;
Chris Wilson3ef94da2009-09-14 16:50:29 +01003745
Chris Wilson6c085a72012-08-20 11:40:46 +02003746 /* if the object is no longer attached, discard its backing storage */
3747 if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL)
Chris Wilson2d7ef392009-09-20 23:13:10 +01003748 i915_gem_object_truncate(obj);
3749
Chris Wilson05394f32010-11-08 19:18:58 +00003750 args->retained = obj->madv != __I915_MADV_PURGED;
Chris Wilsonbb6baf72009-09-22 14:24:13 +01003751
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003752out:
Chris Wilson05394f32010-11-08 19:18:58 +00003753 drm_gem_object_unreference(&obj->base);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003754unlock:
Chris Wilson3ef94da2009-09-14 16:50:29 +01003755 mutex_unlock(&dev->struct_mutex);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01003756 return ret;
Chris Wilson3ef94da2009-09-14 16:50:29 +01003757}
3758
Chris Wilson37e680a2012-06-07 15:38:42 +01003759void i915_gem_object_init(struct drm_i915_gem_object *obj,
3760 const struct drm_i915_gem_object_ops *ops)
Chris Wilson0327d6b2012-08-11 15:41:06 +01003761{
Chris Wilson0327d6b2012-08-11 15:41:06 +01003762 INIT_LIST_HEAD(&obj->mm_list);
3763 INIT_LIST_HEAD(&obj->gtt_list);
3764 INIT_LIST_HEAD(&obj->ring_list);
3765 INIT_LIST_HEAD(&obj->exec_list);
3766
Chris Wilson37e680a2012-06-07 15:38:42 +01003767 obj->ops = ops;
3768
Chris Wilson0327d6b2012-08-11 15:41:06 +01003769 obj->fence_reg = I915_FENCE_REG_NONE;
3770 obj->madv = I915_MADV_WILLNEED;
3771 /* Avoid an unnecessary call to unbind on the first bind. */
3772 obj->map_and_fenceable = true;
3773
3774 i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
3775}
3776
Chris Wilson37e680a2012-06-07 15:38:42 +01003777static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
3778 .get_pages = i915_gem_object_get_pages_gtt,
3779 .put_pages = i915_gem_object_put_pages_gtt,
3780};
3781
Chris Wilson05394f32010-11-08 19:18:58 +00003782struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3783 size_t size)
Daniel Vetterac52bc52010-04-09 19:05:06 +00003784{
Daniel Vetterc397b902010-04-09 19:05:07 +00003785 struct drm_i915_gem_object *obj;
Hugh Dickins5949eac2011-06-27 16:18:18 -07003786 struct address_space *mapping;
Daniel Vetter1a240d42012-11-29 22:18:51 +01003787 gfp_t mask;
Daniel Vetterc397b902010-04-09 19:05:07 +00003788
Chris Wilson42dcedd2012-11-15 11:32:30 +00003789 obj = i915_gem_object_alloc(dev);
Daniel Vetterc397b902010-04-09 19:05:07 +00003790 if (obj == NULL)
3791 return NULL;
3792
3793 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
Chris Wilson42dcedd2012-11-15 11:32:30 +00003794 i915_gem_object_free(obj);
Daniel Vetterc397b902010-04-09 19:05:07 +00003795 return NULL;
3796 }
3797
Chris Wilsonbed1ea92012-05-24 20:48:12 +01003798 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
3799 if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
3800 /* 965gm cannot relocate objects above 4GiB. */
3801 mask &= ~__GFP_HIGHMEM;
3802 mask |= __GFP_DMA32;
3803 }
3804
Al Viro496ad9a2013-01-23 17:07:38 -05003805 mapping = file_inode(obj->base.filp)->i_mapping;
Chris Wilsonbed1ea92012-05-24 20:48:12 +01003806 mapping_set_gfp_mask(mapping, mask);
Hugh Dickins5949eac2011-06-27 16:18:18 -07003807
Chris Wilson37e680a2012-06-07 15:38:42 +01003808 i915_gem_object_init(obj, &i915_gem_object_ops);
Chris Wilson73aa8082010-09-30 11:46:12 +01003809
Daniel Vetterc397b902010-04-09 19:05:07 +00003810 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3811 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3812
Eugeni Dodonov3d29b842012-01-17 14:43:53 -02003813 if (HAS_LLC(dev)) {
3814 /* On some devices, we can have the GPU use the LLC (the CPU
Eric Anholta1871112011-03-29 16:59:55 -07003815 * cache) for about a 10% performance improvement
3816 * compared to uncached. Graphics requests other than
3817 * display scanout are coherent with the CPU in
3818 * accessing this cache. This means in this mode we
3819 * don't need to clflush on the CPU side, and on the
3820 * GPU side we only need to flush internal caches to
3821 * get data visible to the CPU.
3822 *
3823 * However, we maintain the display planes as UC, and so
3824 * need to rebind when first used as such.
3825 */
3826 obj->cache_level = I915_CACHE_LLC;
3827 } else
3828 obj->cache_level = I915_CACHE_NONE;
3829
Chris Wilson05394f32010-11-08 19:18:58 +00003830 return obj;
Daniel Vetterac52bc52010-04-09 19:05:06 +00003831}
3832
Eric Anholt673a3942008-07-30 12:06:12 -07003833int i915_gem_init_object(struct drm_gem_object *obj)
3834{
Daniel Vetterc397b902010-04-09 19:05:07 +00003835 BUG();
Jesse Barnesde151cf2008-11-12 10:03:55 -08003836
Eric Anholt673a3942008-07-30 12:06:12 -07003837 return 0;
3838}
3839
Chris Wilson1488fc02012-04-24 15:47:31 +01003840void i915_gem_free_object(struct drm_gem_object *gem_obj)
Chris Wilsonbe726152010-07-23 23:18:50 +01003841{
Chris Wilson1488fc02012-04-24 15:47:31 +01003842 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
Chris Wilson05394f32010-11-08 19:18:58 +00003843 struct drm_device *dev = obj->base.dev;
Chris Wilsonbe726152010-07-23 23:18:50 +01003844 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsonbe726152010-07-23 23:18:50 +01003845
Chris Wilson26e12f892011-03-20 11:20:19 +00003846 trace_i915_gem_object_destroy(obj);
3847
Chris Wilson1488fc02012-04-24 15:47:31 +01003848 if (obj->phys_obj)
3849 i915_gem_detach_phys_object(dev, obj);
3850
3851 obj->pin_count = 0;
3852 if (WARN_ON(i915_gem_object_unbind(obj) == -ERESTARTSYS)) {
3853 bool was_interruptible;
3854
3855 was_interruptible = dev_priv->mm.interruptible;
3856 dev_priv->mm.interruptible = false;
3857
3858 WARN_ON(i915_gem_object_unbind(obj));
3859
3860 dev_priv->mm.interruptible = was_interruptible;
3861 }
3862
Ben Widawsky1d64ae72013-05-31 14:46:20 -07003863 /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
3864 * before progressing. */
3865 if (obj->stolen)
3866 i915_gem_object_unpin_pages(obj);
3867
Ben Widawsky401c29f2013-05-31 11:28:47 -07003868 if (WARN_ON(obj->pages_pin_count))
3869 obj->pages_pin_count = 0;
Chris Wilson37e680a2012-06-07 15:38:42 +01003870 i915_gem_object_put_pages(obj);
Chris Wilsond8cb5082012-08-11 15:41:03 +01003871 i915_gem_object_free_mmap_offset(obj);
Chris Wilson0104fdb2012-11-15 11:32:26 +00003872 i915_gem_object_release_stolen(obj);
Chris Wilsonbe726152010-07-23 23:18:50 +01003873
Chris Wilson9da3da62012-06-01 15:20:22 +01003874 BUG_ON(obj->pages);
3875
Chris Wilson2f745ad2012-09-04 21:02:58 +01003876 if (obj->base.import_attach)
3877 drm_prime_gem_destroy(&obj->base, NULL);
Chris Wilsonbe726152010-07-23 23:18:50 +01003878
Chris Wilson05394f32010-11-08 19:18:58 +00003879 drm_gem_object_release(&obj->base);
3880 i915_gem_info_remove_obj(dev_priv, obj->base.size);
Chris Wilsonbe726152010-07-23 23:18:50 +01003881
Chris Wilson05394f32010-11-08 19:18:58 +00003882 kfree(obj->bit_17);
Chris Wilson42dcedd2012-11-15 11:32:30 +00003883 i915_gem_object_free(obj);
Chris Wilsonbe726152010-07-23 23:18:50 +01003884}
3885
Jesse Barnes5669fca2009-02-17 15:13:31 -08003886int
Eric Anholt673a3942008-07-30 12:06:12 -07003887i915_gem_idle(struct drm_device *dev)
3888{
3889 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson29105cc2010-01-07 10:39:13 +00003890 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003891
Keith Packard6dbe2772008-10-14 21:41:13 -07003892 mutex_lock(&dev->struct_mutex);
3893
Chris Wilson87acb0a2010-10-19 10:13:00 +01003894 if (dev_priv->mm.suspended) {
Keith Packard6dbe2772008-10-14 21:41:13 -07003895 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07003896 return 0;
Keith Packard6dbe2772008-10-14 21:41:13 -07003897 }
Eric Anholt673a3942008-07-30 12:06:12 -07003898
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07003899 ret = i915_gpu_idle(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07003900 if (ret) {
3901 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07003902 return ret;
Keith Packard6dbe2772008-10-14 21:41:13 -07003903 }
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07003904 i915_gem_retire_requests(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07003905
Chris Wilson29105cc2010-01-07 10:39:13 +00003906 /* Under UMS, be paranoid and evict. */
Chris Wilsona39d7ef2012-04-24 18:22:52 +01003907 if (!drm_core_check_feature(dev, DRIVER_MODESET))
Chris Wilson6c085a72012-08-20 11:40:46 +02003908 i915_gem_evict_everything(dev);
Chris Wilson29105cc2010-01-07 10:39:13 +00003909
Chris Wilson312817a2010-11-22 11:50:11 +00003910 i915_gem_reset_fences(dev);
3911
Chris Wilson29105cc2010-01-07 10:39:13 +00003912 /* Hack! Don't let anybody do execbuf while we don't control the chip.
3913 * We need to replace this with a semaphore, or something.
3914 * And not confound mm.suspended!
3915 */
3916 dev_priv->mm.suspended = 1;
Daniel Vetter99584db2012-11-14 17:14:04 +01003917 del_timer_sync(&dev_priv->gpu_error.hangcheck_timer);
Chris Wilson29105cc2010-01-07 10:39:13 +00003918
3919 i915_kernel_lost_context(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07003920 i915_gem_cleanup_ringbuffer(dev);
Chris Wilson29105cc2010-01-07 10:39:13 +00003921
Keith Packard6dbe2772008-10-14 21:41:13 -07003922 mutex_unlock(&dev->struct_mutex);
3923
Chris Wilson29105cc2010-01-07 10:39:13 +00003924 /* Cancel the retire work handler, which should be idle now. */
3925 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3926
Eric Anholt673a3942008-07-30 12:06:12 -07003927 return 0;
3928}
3929
Ben Widawskyb9524a12012-05-25 16:56:24 -07003930void i915_gem_l3_remap(struct drm_device *dev)
3931{
3932 drm_i915_private_t *dev_priv = dev->dev_private;
3933 u32 misccpctl;
3934 int i;
3935
Daniel Vettereb32e452013-02-14 19:46:07 +01003936 if (!HAS_L3_GPU_CACHE(dev))
Ben Widawskyb9524a12012-05-25 16:56:24 -07003937 return;
3938
Daniel Vettera4da4fa2012-11-02 19:55:07 +01003939 if (!dev_priv->l3_parity.remap_info)
Ben Widawskyb9524a12012-05-25 16:56:24 -07003940 return;
3941
3942 misccpctl = I915_READ(GEN7_MISCCPCTL);
3943 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
3944 POSTING_READ(GEN7_MISCCPCTL);
3945
3946 for (i = 0; i < GEN7_L3LOG_SIZE; i += 4) {
3947 u32 remap = I915_READ(GEN7_L3LOG_BASE + i);
Daniel Vettera4da4fa2012-11-02 19:55:07 +01003948 if (remap && remap != dev_priv->l3_parity.remap_info[i/4])
Ben Widawskyb9524a12012-05-25 16:56:24 -07003949 DRM_DEBUG("0x%x was already programmed to %x\n",
3950 GEN7_L3LOG_BASE + i, remap);
Daniel Vettera4da4fa2012-11-02 19:55:07 +01003951 if (remap && !dev_priv->l3_parity.remap_info[i/4])
Ben Widawskyb9524a12012-05-25 16:56:24 -07003952 DRM_DEBUG_DRIVER("Clearing remapped register\n");
Daniel Vettera4da4fa2012-11-02 19:55:07 +01003953 I915_WRITE(GEN7_L3LOG_BASE + i, dev_priv->l3_parity.remap_info[i/4]);
Ben Widawskyb9524a12012-05-25 16:56:24 -07003954 }
3955
3956 /* Make sure all the writes land before disabling dop clock gating */
3957 POSTING_READ(GEN7_L3LOG_BASE);
3958
3959 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
3960}
3961
Daniel Vetterf691e2f2012-02-02 09:58:12 +01003962void i915_gem_init_swizzling(struct drm_device *dev)
3963{
3964 drm_i915_private_t *dev_priv = dev->dev_private;
3965
Daniel Vetter11782b02012-01-31 16:47:55 +01003966 if (INTEL_INFO(dev)->gen < 5 ||
Daniel Vetterf691e2f2012-02-02 09:58:12 +01003967 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
3968 return;
3969
3970 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
3971 DISP_TILE_SURFACE_SWIZZLING);
3972
Daniel Vetter11782b02012-01-31 16:47:55 +01003973 if (IS_GEN5(dev))
3974 return;
3975
Daniel Vetterf691e2f2012-02-02 09:58:12 +01003976 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
3977 if (IS_GEN6(dev))
Daniel Vetter6b26c862012-04-24 14:04:12 +02003978 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
Ben Widawsky8782e262012-12-18 10:31:23 -08003979 else if (IS_GEN7(dev))
Daniel Vetter6b26c862012-04-24 14:04:12 +02003980 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
Ben Widawsky8782e262012-12-18 10:31:23 -08003981 else
3982 BUG();
Daniel Vetterf691e2f2012-02-02 09:58:12 +01003983}
Daniel Vettere21af882012-02-09 20:53:27 +01003984
Chris Wilson67b1b572012-07-05 23:49:40 +01003985static bool
3986intel_enable_blt(struct drm_device *dev)
3987{
3988 if (!HAS_BLT(dev))
3989 return false;
3990
3991 /* The blitter was dysfunctional on early prototypes */
3992 if (IS_GEN6(dev) && dev->pdev->revision < 8) {
3993 DRM_INFO("BLT not supported on this pre-production hardware;"
3994 " graphics performance will be degraded.\n");
3995 return false;
3996 }
3997
3998 return true;
3999}
4000
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004001static int i915_gem_init_rings(struct drm_device *dev)
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004002{
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004003 struct drm_i915_private *dev_priv = dev->dev_private;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004004 int ret;
Chris Wilson68f95ba2010-05-27 13:18:22 +01004005
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08004006 ret = intel_init_render_ring_buffer(dev);
Chris Wilson68f95ba2010-05-27 13:18:22 +01004007 if (ret)
Chris Wilsonb6913e42010-11-12 10:46:37 +00004008 return ret;
Chris Wilson68f95ba2010-05-27 13:18:22 +01004009
4010 if (HAS_BSD(dev)) {
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08004011 ret = intel_init_bsd_ring_buffer(dev);
Chris Wilson68f95ba2010-05-27 13:18:22 +01004012 if (ret)
4013 goto cleanup_render_ring;
Zou Nan haid1b851f2010-05-21 09:08:57 +08004014 }
Chris Wilson68f95ba2010-05-27 13:18:22 +01004015
Chris Wilson67b1b572012-07-05 23:49:40 +01004016 if (intel_enable_blt(dev)) {
Chris Wilson549f7362010-10-19 11:19:32 +01004017 ret = intel_init_blt_ring_buffer(dev);
4018 if (ret)
4019 goto cleanup_bsd_ring;
4020 }
4021
Ben Widawsky9a8a2212013-05-28 19:22:23 -07004022 if (HAS_VEBOX(dev)) {
4023 ret = intel_init_vebox_ring_buffer(dev);
4024 if (ret)
4025 goto cleanup_blt_ring;
4026 }
4027
4028
Mika Kuoppala99433932013-01-22 14:12:17 +02004029 ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
4030 if (ret)
Ben Widawsky9a8a2212013-05-28 19:22:23 -07004031 goto cleanup_vebox_ring;
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004032
4033 return 0;
4034
Ben Widawsky9a8a2212013-05-28 19:22:23 -07004035cleanup_vebox_ring:
4036 intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004037cleanup_blt_ring:
4038 intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
4039cleanup_bsd_ring:
4040 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4041cleanup_render_ring:
4042 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4043
4044 return ret;
4045}
4046
4047int
4048i915_gem_init_hw(struct drm_device *dev)
4049{
4050 drm_i915_private_t *dev_priv = dev->dev_private;
4051 int ret;
4052
4053 if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
4054 return -EIO;
4055
4056 if (IS_HASWELL(dev) && (I915_READ(0x120010) == 1))
4057 I915_WRITE(0x9008, I915_READ(0x9008) | 0xf0000);
4058
Ben Widawsky88a2b2a2013-04-05 13:12:43 -07004059 if (HAS_PCH_NOP(dev)) {
4060 u32 temp = I915_READ(GEN7_MSG_CTL);
4061 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4062 I915_WRITE(GEN7_MSG_CTL, temp);
4063 }
4064
Ben Widawsky4fc7c972013-02-08 11:49:24 -08004065 i915_gem_l3_remap(dev);
4066
4067 i915_gem_init_swizzling(dev);
4068
4069 ret = i915_gem_init_rings(dev);
4070 if (ret)
Mika Kuoppala99433932013-01-22 14:12:17 +02004071 return ret;
4072
Ben Widawsky254f9652012-06-04 14:42:42 -07004073 /*
4074 * XXX: There was some w/a described somewhere suggesting loading
4075 * contexts before PPGTT.
4076 */
4077 i915_gem_context_init(dev);
Ben Widawskyb7c36d22013-04-08 18:43:56 -07004078 if (dev_priv->mm.aliasing_ppgtt) {
4079 ret = dev_priv->mm.aliasing_ppgtt->enable(dev);
4080 if (ret) {
4081 i915_gem_cleanup_aliasing_ppgtt(dev);
4082 DRM_INFO("PPGTT enable failed. This is not fatal, but unexpected\n");
4083 }
4084 }
Daniel Vettere21af882012-02-09 20:53:27 +01004085
Chris Wilson68f95ba2010-05-27 13:18:22 +01004086 return 0;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004087}
4088
Chris Wilson1070a422012-04-24 15:47:41 +01004089int i915_gem_init(struct drm_device *dev)
4090{
4091 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson1070a422012-04-24 15:47:41 +01004092 int ret;
4093
Chris Wilson1070a422012-04-24 15:47:41 +01004094 mutex_lock(&dev->struct_mutex);
Jesse Barnesd62b4892013-03-08 10:45:53 -08004095
4096 if (IS_VALLEYVIEW(dev)) {
4097 /* VLVA0 (potential hack), BIOS isn't actually waking us */
4098 I915_WRITE(VLV_GTLC_WAKE_CTRL, 1);
4099 if (wait_for((I915_READ(VLV_GTLC_PW_STATUS) & 1) == 1, 10))
4100 DRM_DEBUG_DRIVER("allow wake ack timed out\n");
4101 }
4102
Ben Widawskyd7e50082012-12-18 10:31:25 -08004103 i915_gem_init_global_gtt(dev);
Jesse Barnesd62b4892013-03-08 10:45:53 -08004104
Chris Wilson1070a422012-04-24 15:47:41 +01004105 ret = i915_gem_init_hw(dev);
4106 mutex_unlock(&dev->struct_mutex);
4107 if (ret) {
4108 i915_gem_cleanup_aliasing_ppgtt(dev);
4109 return ret;
4110 }
4111
Daniel Vetter53ca26c2012-04-26 23:28:03 +02004112 /* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
4113 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4114 dev_priv->dri1.allow_batchbuffer = 1;
Chris Wilson1070a422012-04-24 15:47:41 +01004115 return 0;
4116}
4117
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004118void
4119i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4120{
4121 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsonb4519512012-05-11 14:29:30 +01004122 struct intel_ring_buffer *ring;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00004123 int i;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004124
Chris Wilsonb4519512012-05-11 14:29:30 +01004125 for_each_ring(ring, dev_priv, i)
4126 intel_cleanup_ring_buffer(ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004127}
4128
4129int
Eric Anholt673a3942008-07-30 12:06:12 -07004130i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4131 struct drm_file *file_priv)
4132{
4133 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsonb4519512012-05-11 14:29:30 +01004134 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004135
Jesse Barnes79e53942008-11-07 14:24:08 -08004136 if (drm_core_check_feature(dev, DRIVER_MODESET))
4137 return 0;
4138
Daniel Vetter1f83fee2012-11-15 17:17:22 +01004139 if (i915_reset_in_progress(&dev_priv->gpu_error)) {
Eric Anholt673a3942008-07-30 12:06:12 -07004140 DRM_ERROR("Reenabling wedged hardware, good luck\n");
Daniel Vetter1f83fee2012-11-15 17:17:22 +01004141 atomic_set(&dev_priv->gpu_error.reset_counter, 0);
Eric Anholt673a3942008-07-30 12:06:12 -07004142 }
4143
Eric Anholt673a3942008-07-30 12:06:12 -07004144 mutex_lock(&dev->struct_mutex);
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004145 dev_priv->mm.suspended = 0;
4146
Daniel Vetterf691e2f2012-02-02 09:58:12 +01004147 ret = i915_gem_init_hw(dev);
Wu Fengguangd816f6a2009-04-18 10:43:32 +08004148 if (ret != 0) {
4149 mutex_unlock(&dev->struct_mutex);
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004150 return ret;
Wu Fengguangd816f6a2009-04-18 10:43:32 +08004151 }
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004152
Chris Wilson69dc4982010-10-19 10:36:51 +01004153 BUG_ON(!list_empty(&dev_priv->mm.active_list));
Eric Anholt673a3942008-07-30 12:06:12 -07004154 mutex_unlock(&dev->struct_mutex);
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004155
Chris Wilson5f353082010-06-07 14:03:03 +01004156 ret = drm_irq_install(dev);
4157 if (ret)
4158 goto cleanup_ringbuffer;
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004159
Eric Anholt673a3942008-07-30 12:06:12 -07004160 return 0;
Chris Wilson5f353082010-06-07 14:03:03 +01004161
4162cleanup_ringbuffer:
4163 mutex_lock(&dev->struct_mutex);
4164 i915_gem_cleanup_ringbuffer(dev);
4165 dev_priv->mm.suspended = 1;
4166 mutex_unlock(&dev->struct_mutex);
4167
4168 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004169}
4170
4171int
4172i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4173 struct drm_file *file_priv)
4174{
Jesse Barnes79e53942008-11-07 14:24:08 -08004175 if (drm_core_check_feature(dev, DRIVER_MODESET))
4176 return 0;
4177
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004178 drm_irq_uninstall(dev);
Linus Torvaldse6890f62009-09-08 17:09:24 -07004179 return i915_gem_idle(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07004180}
4181
4182void
4183i915_gem_lastclose(struct drm_device *dev)
4184{
4185 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004186
Eric Anholte806b492009-01-22 09:56:58 -08004187 if (drm_core_check_feature(dev, DRIVER_MODESET))
4188 return;
4189
Keith Packard6dbe2772008-10-14 21:41:13 -07004190 ret = i915_gem_idle(dev);
4191 if (ret)
4192 DRM_ERROR("failed to idle hardware: %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07004193}
4194
Chris Wilson64193402010-10-24 12:38:05 +01004195static void
4196init_ring_lists(struct intel_ring_buffer *ring)
4197{
4198 INIT_LIST_HEAD(&ring->active_list);
4199 INIT_LIST_HEAD(&ring->request_list);
Chris Wilson64193402010-10-24 12:38:05 +01004200}
4201
Eric Anholt673a3942008-07-30 12:06:12 -07004202void
4203i915_gem_load(struct drm_device *dev)
4204{
4205 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson42dcedd2012-11-15 11:32:30 +00004206 int i;
4207
4208 dev_priv->slab =
4209 kmem_cache_create("i915_gem_object",
4210 sizeof(struct drm_i915_gem_object), 0,
4211 SLAB_HWCACHE_ALIGN,
4212 NULL);
Eric Anholt673a3942008-07-30 12:06:12 -07004213
Chris Wilson69dc4982010-10-19 10:36:51 +01004214 INIT_LIST_HEAD(&dev_priv->mm.active_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004215 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
Chris Wilson6c085a72012-08-20 11:40:46 +02004216 INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
4217 INIT_LIST_HEAD(&dev_priv->mm.bound_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07004218 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00004219 for (i = 0; i < I915_NUM_RINGS; i++)
4220 init_ring_lists(&dev_priv->ring[i]);
Daniel Vetter4b9de732011-10-09 21:52:02 +02004221 for (i = 0; i < I915_MAX_NUM_FENCES; i++)
Daniel Vetter007cc8a2010-04-28 11:02:31 +02004222 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004223 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4224 i915_gem_retire_work_handler);
Daniel Vetter1f83fee2012-11-15 17:17:22 +01004225 init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
Chris Wilson31169712009-09-14 16:50:28 +01004226
Dave Airlie94400122010-07-20 13:15:31 +10004227 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4228 if (IS_GEN3(dev)) {
Daniel Vetter50743292012-04-26 22:02:54 +02004229 I915_WRITE(MI_ARB_STATE,
4230 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
Dave Airlie94400122010-07-20 13:15:31 +10004231 }
4232
Chris Wilson72bfa192010-12-19 11:42:05 +00004233 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
4234
Jesse Barnesde151cf2008-11-12 10:03:55 -08004235 /* Old X drivers will take 0-2 for front, back, depth buffers */
Eric Anholtb397c832010-01-26 09:43:10 -08004236 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4237 dev_priv->fence_reg_start = 3;
Jesse Barnesde151cf2008-11-12 10:03:55 -08004238
Ville Syrjälä42b5aea2013-04-09 13:02:47 +03004239 if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev))
4240 dev_priv->num_fence_regs = 32;
4241 else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
Jesse Barnesde151cf2008-11-12 10:03:55 -08004242 dev_priv->num_fence_regs = 16;
4243 else
4244 dev_priv->num_fence_regs = 8;
4245
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004246 /* Initialize fence registers to zero */
Chris Wilsonada726c2012-04-17 15:31:32 +01004247 i915_gem_reset_fences(dev);
Eric Anholt10ed13e2011-05-06 13:53:49 -07004248
Eric Anholt673a3942008-07-30 12:06:12 -07004249 i915_gem_detect_bit_6_swizzle(dev);
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05004250 init_waitqueue_head(&dev_priv->pending_flip_queue);
Chris Wilson17250b72010-10-28 12:51:39 +01004251
Chris Wilsonce453d82011-02-21 14:43:56 +00004252 dev_priv->mm.interruptible = true;
4253
Chris Wilson17250b72010-10-28 12:51:39 +01004254 dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
4255 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4256 register_shrinker(&dev_priv->mm.inactive_shrinker);
Eric Anholt673a3942008-07-30 12:06:12 -07004257}
Dave Airlie71acb5e2008-12-30 20:31:46 +10004258
4259/*
4260 * Create a physically contiguous memory object for this object
4261 * e.g. for cursor + overlay regs
4262 */
Chris Wilson995b6762010-08-20 13:23:26 +01004263static int i915_gem_init_phys_object(struct drm_device *dev,
4264 int id, int size, int align)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004265{
4266 drm_i915_private_t *dev_priv = dev->dev_private;
4267 struct drm_i915_gem_phys_object *phys_obj;
4268 int ret;
4269
4270 if (dev_priv->mm.phys_objs[id - 1] || !size)
4271 return 0;
4272
Eric Anholt9a298b22009-03-24 12:23:04 -07004273 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004274 if (!phys_obj)
4275 return -ENOMEM;
4276
4277 phys_obj->id = id;
4278
Chris Wilson6eeefaf2010-08-07 11:01:39 +01004279 phys_obj->handle = drm_pci_alloc(dev, size, align);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004280 if (!phys_obj->handle) {
4281 ret = -ENOMEM;
4282 goto kfree_obj;
4283 }
4284#ifdef CONFIG_X86
4285 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4286#endif
4287
4288 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4289
4290 return 0;
4291kfree_obj:
Eric Anholt9a298b22009-03-24 12:23:04 -07004292 kfree(phys_obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004293 return ret;
4294}
4295
Chris Wilson995b6762010-08-20 13:23:26 +01004296static void i915_gem_free_phys_object(struct drm_device *dev, int id)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004297{
4298 drm_i915_private_t *dev_priv = dev->dev_private;
4299 struct drm_i915_gem_phys_object *phys_obj;
4300
4301 if (!dev_priv->mm.phys_objs[id - 1])
4302 return;
4303
4304 phys_obj = dev_priv->mm.phys_objs[id - 1];
4305 if (phys_obj->cur_obj) {
4306 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4307 }
4308
4309#ifdef CONFIG_X86
4310 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4311#endif
4312 drm_pci_free(dev, phys_obj->handle);
4313 kfree(phys_obj);
4314 dev_priv->mm.phys_objs[id - 1] = NULL;
4315}
4316
4317void i915_gem_free_all_phys_object(struct drm_device *dev)
4318{
4319 int i;
4320
Dave Airlie260883c2009-01-22 17:58:49 +10004321 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004322 i915_gem_free_phys_object(dev, i);
4323}
4324
4325void i915_gem_detach_phys_object(struct drm_device *dev,
Chris Wilson05394f32010-11-08 19:18:58 +00004326 struct drm_i915_gem_object *obj)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004327{
Al Viro496ad9a2013-01-23 17:07:38 -05004328 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
Chris Wilsone5281cc2010-10-28 13:45:36 +01004329 char *vaddr;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004330 int i;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004331 int page_count;
4332
Chris Wilson05394f32010-11-08 19:18:58 +00004333 if (!obj->phys_obj)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004334 return;
Chris Wilson05394f32010-11-08 19:18:58 +00004335 vaddr = obj->phys_obj->handle->vaddr;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004336
Chris Wilson05394f32010-11-08 19:18:58 +00004337 page_count = obj->base.size / PAGE_SIZE;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004338 for (i = 0; i < page_count; i++) {
Hugh Dickins5949eac2011-06-27 16:18:18 -07004339 struct page *page = shmem_read_mapping_page(mapping, i);
Chris Wilsone5281cc2010-10-28 13:45:36 +01004340 if (!IS_ERR(page)) {
4341 char *dst = kmap_atomic(page);
4342 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4343 kunmap_atomic(dst);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004344
Chris Wilsone5281cc2010-10-28 13:45:36 +01004345 drm_clflush_pages(&page, 1);
4346
4347 set_page_dirty(page);
4348 mark_page_accessed(page);
4349 page_cache_release(page);
4350 }
Dave Airlie71acb5e2008-12-30 20:31:46 +10004351 }
Ben Widawskye76e9ae2012-11-04 09:21:27 -08004352 i915_gem_chipset_flush(dev);
Chris Wilsond78b47b2009-06-17 21:52:49 +01004353
Chris Wilson05394f32010-11-08 19:18:58 +00004354 obj->phys_obj->cur_obj = NULL;
4355 obj->phys_obj = NULL;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004356}
4357
4358int
4359i915_gem_attach_phys_object(struct drm_device *dev,
Chris Wilson05394f32010-11-08 19:18:58 +00004360 struct drm_i915_gem_object *obj,
Chris Wilson6eeefaf2010-08-07 11:01:39 +01004361 int id,
4362 int align)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004363{
Al Viro496ad9a2013-01-23 17:07:38 -05004364 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004365 drm_i915_private_t *dev_priv = dev->dev_private;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004366 int ret = 0;
4367 int page_count;
4368 int i;
4369
4370 if (id > I915_MAX_PHYS_OBJECT)
4371 return -EINVAL;
4372
Chris Wilson05394f32010-11-08 19:18:58 +00004373 if (obj->phys_obj) {
4374 if (obj->phys_obj->id == id)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004375 return 0;
4376 i915_gem_detach_phys_object(dev, obj);
4377 }
4378
Dave Airlie71acb5e2008-12-30 20:31:46 +10004379 /* create a new object */
4380 if (!dev_priv->mm.phys_objs[id - 1]) {
4381 ret = i915_gem_init_phys_object(dev, id,
Chris Wilson05394f32010-11-08 19:18:58 +00004382 obj->base.size, align);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004383 if (ret) {
Chris Wilson05394f32010-11-08 19:18:58 +00004384 DRM_ERROR("failed to init phys object %d size: %zu\n",
4385 id, obj->base.size);
Chris Wilsone5281cc2010-10-28 13:45:36 +01004386 return ret;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004387 }
4388 }
4389
4390 /* bind to the object */
Chris Wilson05394f32010-11-08 19:18:58 +00004391 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4392 obj->phys_obj->cur_obj = obj;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004393
Chris Wilson05394f32010-11-08 19:18:58 +00004394 page_count = obj->base.size / PAGE_SIZE;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004395
4396 for (i = 0; i < page_count; i++) {
Chris Wilsone5281cc2010-10-28 13:45:36 +01004397 struct page *page;
4398 char *dst, *src;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004399
Hugh Dickins5949eac2011-06-27 16:18:18 -07004400 page = shmem_read_mapping_page(mapping, i);
Chris Wilsone5281cc2010-10-28 13:45:36 +01004401 if (IS_ERR(page))
4402 return PTR_ERR(page);
4403
Chris Wilsonff75b9b2010-10-30 22:52:31 +01004404 src = kmap_atomic(page);
Chris Wilson05394f32010-11-08 19:18:58 +00004405 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004406 memcpy(dst, src, PAGE_SIZE);
Peter Zijlstra3e4d3af2010-10-26 14:21:51 -07004407 kunmap_atomic(src);
Chris Wilsone5281cc2010-10-28 13:45:36 +01004408
4409 mark_page_accessed(page);
4410 page_cache_release(page);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004411 }
4412
4413 return 0;
Dave Airlie71acb5e2008-12-30 20:31:46 +10004414}
4415
4416static int
Chris Wilson05394f32010-11-08 19:18:58 +00004417i915_gem_phys_pwrite(struct drm_device *dev,
4418 struct drm_i915_gem_object *obj,
Dave Airlie71acb5e2008-12-30 20:31:46 +10004419 struct drm_i915_gem_pwrite *args,
4420 struct drm_file *file_priv)
4421{
Chris Wilson05394f32010-11-08 19:18:58 +00004422 void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
Ville Syrjälä2bb46292013-02-22 16:12:51 +02004423 char __user *user_data = to_user_ptr(args->data_ptr);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004424
Chris Wilsonb47b30c2010-11-08 01:12:29 +00004425 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4426 unsigned long unwritten;
4427
4428 /* The physical object once assigned is fixed for the lifetime
4429 * of the obj, so we can safely drop the lock and continue
4430 * to access vaddr.
4431 */
4432 mutex_unlock(&dev->struct_mutex);
4433 unwritten = copy_from_user(vaddr, user_data, args->size);
4434 mutex_lock(&dev->struct_mutex);
4435 if (unwritten)
4436 return -EFAULT;
4437 }
Dave Airlie71acb5e2008-12-30 20:31:46 +10004438
Ben Widawskye76e9ae2012-11-04 09:21:27 -08004439 i915_gem_chipset_flush(dev);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004440 return 0;
4441}
Eric Anholtb9624422009-06-03 07:27:35 +00004442
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004443void i915_gem_release(struct drm_device *dev, struct drm_file *file)
Eric Anholtb9624422009-06-03 07:27:35 +00004444{
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004445 struct drm_i915_file_private *file_priv = file->driver_priv;
Eric Anholtb9624422009-06-03 07:27:35 +00004446
4447 /* Clean up our request list when the client is going away, so that
4448 * later retire_requests won't dereference our soon-to-be-gone
4449 * file_priv.
4450 */
Chris Wilson1c255952010-09-26 11:03:27 +01004451 spin_lock(&file_priv->mm.lock);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004452 while (!list_empty(&file_priv->mm.request_list)) {
4453 struct drm_i915_gem_request *request;
4454
4455 request = list_first_entry(&file_priv->mm.request_list,
4456 struct drm_i915_gem_request,
4457 client_list);
4458 list_del(&request->client_list);
4459 request->file_priv = NULL;
4460 }
Chris Wilson1c255952010-09-26 11:03:27 +01004461 spin_unlock(&file_priv->mm.lock);
Eric Anholtb9624422009-06-03 07:27:35 +00004462}
Chris Wilson31169712009-09-14 16:50:28 +01004463
Chris Wilson57745062012-11-21 13:04:04 +00004464static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
4465{
4466 if (!mutex_is_locked(mutex))
4467 return false;
4468
4469#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
4470 return mutex->owner == task;
4471#else
4472 /* Since UP may be pre-empted, we cannot assume that we own the lock */
4473 return false;
4474#endif
4475}
4476
Chris Wilson31169712009-09-14 16:50:28 +01004477static int
Ying Han1495f232011-05-24 17:12:27 -07004478i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
Chris Wilson31169712009-09-14 16:50:28 +01004479{
Chris Wilson17250b72010-10-28 12:51:39 +01004480 struct drm_i915_private *dev_priv =
4481 container_of(shrinker,
4482 struct drm_i915_private,
4483 mm.inactive_shrinker);
4484 struct drm_device *dev = dev_priv->dev;
Chris Wilson6c085a72012-08-20 11:40:46 +02004485 struct drm_i915_gem_object *obj;
Ying Han1495f232011-05-24 17:12:27 -07004486 int nr_to_scan = sc->nr_to_scan;
Chris Wilson57745062012-11-21 13:04:04 +00004487 bool unlock = true;
Chris Wilson17250b72010-10-28 12:51:39 +01004488 int cnt;
4489
Chris Wilson57745062012-11-21 13:04:04 +00004490 if (!mutex_trylock(&dev->struct_mutex)) {
4491 if (!mutex_is_locked_by(&dev->struct_mutex, current))
4492 return 0;
4493
Daniel Vetter677feac2012-12-19 14:33:45 +01004494 if (dev_priv->mm.shrinker_no_lock_stealing)
4495 return 0;
4496
Chris Wilson57745062012-11-21 13:04:04 +00004497 unlock = false;
4498 }
Chris Wilson31169712009-09-14 16:50:28 +01004499
Chris Wilson6c085a72012-08-20 11:40:46 +02004500 if (nr_to_scan) {
4501 nr_to_scan -= i915_gem_purge(dev_priv, nr_to_scan);
4502 if (nr_to_scan > 0)
Daniel Vetter93927ca2013-01-10 18:03:00 +01004503 nr_to_scan -= __i915_gem_shrink(dev_priv, nr_to_scan,
4504 false);
4505 if (nr_to_scan > 0)
Chris Wilson6c085a72012-08-20 11:40:46 +02004506 i915_gem_shrink_all(dev_priv);
Chris Wilson31169712009-09-14 16:50:28 +01004507 }
4508
Chris Wilson17250b72010-10-28 12:51:39 +01004509 cnt = 0;
Chris Wilson6c085a72012-08-20 11:40:46 +02004510 list_for_each_entry(obj, &dev_priv->mm.unbound_list, gtt_list)
Chris Wilsona5570172012-09-04 21:02:54 +01004511 if (obj->pages_pin_count == 0)
4512 cnt += obj->base.size >> PAGE_SHIFT;
Daniel Vetter93927ca2013-01-10 18:03:00 +01004513 list_for_each_entry(obj, &dev_priv->mm.inactive_list, gtt_list)
Chris Wilsona5570172012-09-04 21:02:54 +01004514 if (obj->pin_count == 0 && obj->pages_pin_count == 0)
Chris Wilson6c085a72012-08-20 11:40:46 +02004515 cnt += obj->base.size >> PAGE_SHIFT;
Chris Wilson31169712009-09-14 16:50:28 +01004516
Chris Wilson57745062012-11-21 13:04:04 +00004517 if (unlock)
4518 mutex_unlock(&dev->struct_mutex);
Chris Wilson6c085a72012-08-20 11:40:46 +02004519 return cnt;
Chris Wilson31169712009-09-14 16:50:28 +01004520}