blob: b52f47af41f499758b58dc9fc8069b20aba30091 [file] [log] [blame]
Eric Anholt673a3942008-07-30 12:06:12 -07001/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
Chris Wilson1c5d22f2009-08-25 11:15:50 +010032#include "i915_trace.h"
Jesse Barnes652c3932009-08-17 13:31:43 -070033#include "intel_drv.h"
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090034#include <linux/slab.h>
Eric Anholt673a3942008-07-30 12:06:12 -070035#include <linux/swap.h>
Jesse Barnes79e53942008-11-07 14:24:08 -080036#include <linux/pci.h>
Zhenyu Wangf8f235e2010-08-27 11:08:57 +080037#include <linux/intel-gtt.h>
Eric Anholt673a3942008-07-30 12:06:12 -070038
Daniel Vetter0108a3e2010-08-07 11:01:21 +010039static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
Chris Wilson2dafb1e2010-06-07 14:03:05 +010040static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
Eric Anholte47c68e2008-11-14 13:35:19 -080041static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
42static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
Eric Anholte47c68e2008-11-14 13:35:19 -080043static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
44 int write);
45static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
46 uint64_t offset,
47 uint64_t size);
48static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
Daniel Vettere35a41d2010-02-11 22:13:59 +010049static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
50 bool interruptible);
Jesse Barnesde151cf2008-11-12 10:03:55 -080051static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
52 unsigned alignment);
Jesse Barnesde151cf2008-11-12 10:03:55 -080053static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +100054static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
55 struct drm_i915_gem_pwrite *args,
56 struct drm_file *file_priv);
Chris Wilsonbe726152010-07-23 23:18:50 +010057static void i915_gem_free_object_tail(struct drm_gem_object *obj);
Eric Anholt673a3942008-07-30 12:06:12 -070058
Chris Wilson31169712009-09-14 16:50:28 +010059static LIST_HEAD(shrink_list);
60static DEFINE_SPINLOCK(shrink_list_lock);
61
Chris Wilson7d1c4802010-08-07 21:45:03 +010062static inline bool
63i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
64{
65 return obj_priv->gtt_space &&
66 !obj_priv->active &&
67 obj_priv->pin_count == 0;
68}
69
Jesse Barnes79e53942008-11-07 14:24:08 -080070int i915_gem_do_init(struct drm_device *dev, unsigned long start,
71 unsigned long end)
72{
73 drm_i915_private_t *dev_priv = dev->dev_private;
74
75 if (start >= end ||
76 (start & (PAGE_SIZE - 1)) != 0 ||
77 (end & (PAGE_SIZE - 1)) != 0) {
78 return -EINVAL;
79 }
80
81 drm_mm_init(&dev_priv->mm.gtt_space, start,
82 end - start);
83
84 dev->gtt_total = (uint32_t) (end - start);
85
86 return 0;
87}
Keith Packard6dbe2772008-10-14 21:41:13 -070088
Eric Anholt673a3942008-07-30 12:06:12 -070089int
90i915_gem_init_ioctl(struct drm_device *dev, void *data,
91 struct drm_file *file_priv)
92{
Eric Anholt673a3942008-07-30 12:06:12 -070093 struct drm_i915_gem_init *args = data;
Jesse Barnes79e53942008-11-07 14:24:08 -080094 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -070095
96 mutex_lock(&dev->struct_mutex);
Jesse Barnes79e53942008-11-07 14:24:08 -080097 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
Eric Anholt673a3942008-07-30 12:06:12 -070098 mutex_unlock(&dev->struct_mutex);
99
Jesse Barnes79e53942008-11-07 14:24:08 -0800100 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700101}
102
Eric Anholt5a125c32008-10-22 21:40:13 -0700103int
104i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
105 struct drm_file *file_priv)
106{
Eric Anholt5a125c32008-10-22 21:40:13 -0700107 struct drm_i915_gem_get_aperture *args = data;
Eric Anholt5a125c32008-10-22 21:40:13 -0700108
109 if (!(dev->driver->driver_features & DRIVER_GEM))
110 return -ENODEV;
111
112 args->aper_size = dev->gtt_total;
Keith Packard2678d9d2008-11-20 22:54:54 -0800113 args->aper_available_size = (args->aper_size -
114 atomic_read(&dev->pin_memory));
Eric Anholt5a125c32008-10-22 21:40:13 -0700115
116 return 0;
117}
118
Eric Anholt673a3942008-07-30 12:06:12 -0700119
120/**
121 * Creates a new mm object and returns a handle to it.
122 */
123int
124i915_gem_create_ioctl(struct drm_device *dev, void *data,
125 struct drm_file *file_priv)
126{
127 struct drm_i915_gem_create *args = data;
128 struct drm_gem_object *obj;
Pekka Paalanena1a2d1d2009-08-23 12:40:55 +0300129 int ret;
130 u32 handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700131
132 args->size = roundup(args->size, PAGE_SIZE);
133
134 /* Allocate the new object */
Daniel Vetterac52bc52010-04-09 19:05:06 +0000135 obj = i915_gem_alloc_object(dev, args->size);
Eric Anholt673a3942008-07-30 12:06:12 -0700136 if (obj == NULL)
137 return -ENOMEM;
138
139 ret = drm_gem_handle_create(file_priv, obj, &handle);
Chris Wilson1dfd9752010-09-06 14:44:14 +0100140 if (ret) {
141 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700142 return ret;
Chris Wilson1dfd9752010-09-06 14:44:14 +0100143 }
144
145 /* Sink the floating reference from kref_init(handlecount) */
146 drm_gem_object_handle_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700147
148 args->handle = handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700149 return 0;
150}
151
Eric Anholt40123c12009-03-09 13:42:30 -0700152static inline int
Eric Anholteb014592009-03-10 11:44:52 -0700153fast_shmem_read(struct page **pages,
154 loff_t page_base, int page_offset,
155 char __user *data,
156 int length)
157{
158 char __iomem *vaddr;
Florian Mickler2bc43b52009-04-06 22:55:41 +0200159 int unwritten;
Eric Anholteb014592009-03-10 11:44:52 -0700160
161 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
162 if (vaddr == NULL)
163 return -ENOMEM;
Florian Mickler2bc43b52009-04-06 22:55:41 +0200164 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
Eric Anholteb014592009-03-10 11:44:52 -0700165 kunmap_atomic(vaddr, KM_USER0);
166
Florian Mickler2bc43b52009-04-06 22:55:41 +0200167 if (unwritten)
168 return -EFAULT;
169
170 return 0;
Eric Anholteb014592009-03-10 11:44:52 -0700171}
172
Eric Anholt280b7132009-03-12 16:56:27 -0700173static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
174{
175 drm_i915_private_t *dev_priv = obj->dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +0100176 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt280b7132009-03-12 16:56:27 -0700177
178 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
179 obj_priv->tiling_mode != I915_TILING_NONE;
180}
181
Chris Wilson99a03df2010-05-27 14:15:34 +0100182static inline void
Eric Anholt40123c12009-03-09 13:42:30 -0700183slow_shmem_copy(struct page *dst_page,
184 int dst_offset,
185 struct page *src_page,
186 int src_offset,
187 int length)
188{
189 char *dst_vaddr, *src_vaddr;
190
Chris Wilson99a03df2010-05-27 14:15:34 +0100191 dst_vaddr = kmap(dst_page);
192 src_vaddr = kmap(src_page);
Eric Anholt40123c12009-03-09 13:42:30 -0700193
194 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
195
Chris Wilson99a03df2010-05-27 14:15:34 +0100196 kunmap(src_page);
197 kunmap(dst_page);
Eric Anholt40123c12009-03-09 13:42:30 -0700198}
199
Chris Wilson99a03df2010-05-27 14:15:34 +0100200static inline void
Eric Anholt280b7132009-03-12 16:56:27 -0700201slow_shmem_bit17_copy(struct page *gpu_page,
202 int gpu_offset,
203 struct page *cpu_page,
204 int cpu_offset,
205 int length,
206 int is_read)
207{
208 char *gpu_vaddr, *cpu_vaddr;
209
210 /* Use the unswizzled path if this page isn't affected. */
211 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
212 if (is_read)
213 return slow_shmem_copy(cpu_page, cpu_offset,
214 gpu_page, gpu_offset, length);
215 else
216 return slow_shmem_copy(gpu_page, gpu_offset,
217 cpu_page, cpu_offset, length);
218 }
219
Chris Wilson99a03df2010-05-27 14:15:34 +0100220 gpu_vaddr = kmap(gpu_page);
221 cpu_vaddr = kmap(cpu_page);
Eric Anholt280b7132009-03-12 16:56:27 -0700222
223 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
224 * XORing with the other bits (A9 for Y, A9 and A10 for X)
225 */
226 while (length > 0) {
227 int cacheline_end = ALIGN(gpu_offset + 1, 64);
228 int this_length = min(cacheline_end - gpu_offset, length);
229 int swizzled_gpu_offset = gpu_offset ^ 64;
230
231 if (is_read) {
232 memcpy(cpu_vaddr + cpu_offset,
233 gpu_vaddr + swizzled_gpu_offset,
234 this_length);
235 } else {
236 memcpy(gpu_vaddr + swizzled_gpu_offset,
237 cpu_vaddr + cpu_offset,
238 this_length);
239 }
240 cpu_offset += this_length;
241 gpu_offset += this_length;
242 length -= this_length;
243 }
244
Chris Wilson99a03df2010-05-27 14:15:34 +0100245 kunmap(cpu_page);
246 kunmap(gpu_page);
Eric Anholt280b7132009-03-12 16:56:27 -0700247}
248
Eric Anholt673a3942008-07-30 12:06:12 -0700249/**
Eric Anholteb014592009-03-10 11:44:52 -0700250 * This is the fast shmem pread path, which attempts to copy_from_user directly
251 * from the backing pages of the object to the user's address space. On a
252 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
253 */
254static int
255i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
256 struct drm_i915_gem_pread *args,
257 struct drm_file *file_priv)
258{
Daniel Vetter23010e42010-03-08 13:35:02 +0100259 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700260 ssize_t remain;
261 loff_t offset, page_base;
262 char __user *user_data;
263 int page_offset, page_length;
264 int ret;
265
266 user_data = (char __user *) (uintptr_t) args->data_ptr;
267 remain = args->size;
268
269 mutex_lock(&dev->struct_mutex);
270
Chris Wilson4bdadb92010-01-27 13:36:32 +0000271 ret = i915_gem_object_get_pages(obj, 0);
Eric Anholteb014592009-03-10 11:44:52 -0700272 if (ret != 0)
273 goto fail_unlock;
274
275 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
276 args->size);
277 if (ret != 0)
278 goto fail_put_pages;
279
Daniel Vetter23010e42010-03-08 13:35:02 +0100280 obj_priv = to_intel_bo(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700281 offset = args->offset;
282
283 while (remain > 0) {
284 /* Operation in this page
285 *
286 * page_base = page offset within aperture
287 * page_offset = offset within page
288 * page_length = bytes to copy for this page
289 */
290 page_base = (offset & ~(PAGE_SIZE-1));
291 page_offset = offset & (PAGE_SIZE-1);
292 page_length = remain;
293 if ((page_offset + remain) > PAGE_SIZE)
294 page_length = PAGE_SIZE - page_offset;
295
296 ret = fast_shmem_read(obj_priv->pages,
297 page_base, page_offset,
298 user_data, page_length);
299 if (ret)
300 goto fail_put_pages;
301
302 remain -= page_length;
303 user_data += page_length;
304 offset += page_length;
305 }
306
307fail_put_pages:
308 i915_gem_object_put_pages(obj);
309fail_unlock:
310 mutex_unlock(&dev->struct_mutex);
311
312 return ret;
313}
314
Chris Wilson07f73f62009-09-14 16:50:30 +0100315static int
316i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
317{
318 int ret;
319
Chris Wilson4bdadb92010-01-27 13:36:32 +0000320 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
Chris Wilson07f73f62009-09-14 16:50:30 +0100321
322 /* If we've insufficient memory to map in the pages, attempt
323 * to make some space by throwing out some old buffers.
324 */
325 if (ret == -ENOMEM) {
326 struct drm_device *dev = obj->dev;
Chris Wilson07f73f62009-09-14 16:50:30 +0100327
Daniel Vetter0108a3e2010-08-07 11:01:21 +0100328 ret = i915_gem_evict_something(dev, obj->size,
329 i915_gem_get_gtt_alignment(obj));
Chris Wilson07f73f62009-09-14 16:50:30 +0100330 if (ret)
331 return ret;
332
Chris Wilson4bdadb92010-01-27 13:36:32 +0000333 ret = i915_gem_object_get_pages(obj, 0);
Chris Wilson07f73f62009-09-14 16:50:30 +0100334 }
335
336 return ret;
337}
338
Eric Anholteb014592009-03-10 11:44:52 -0700339/**
340 * This is the fallback shmem pread path, which allocates temporary storage
341 * in kernel space to copy_to_user into outside of the struct_mutex, so we
342 * can copy out of the object's backing pages while holding the struct mutex
343 * and not take page faults.
344 */
345static int
346i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
347 struct drm_i915_gem_pread *args,
348 struct drm_file *file_priv)
349{
Daniel Vetter23010e42010-03-08 13:35:02 +0100350 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700351 struct mm_struct *mm = current->mm;
352 struct page **user_pages;
353 ssize_t remain;
354 loff_t offset, pinned_pages, i;
355 loff_t first_data_page, last_data_page, num_pages;
356 int shmem_page_index, shmem_page_offset;
357 int data_page_index, data_page_offset;
358 int page_length;
359 int ret;
360 uint64_t data_ptr = args->data_ptr;
Eric Anholt280b7132009-03-12 16:56:27 -0700361 int do_bit17_swizzling;
Eric Anholteb014592009-03-10 11:44:52 -0700362
363 remain = args->size;
364
365 /* Pin the user pages containing the data. We can't fault while
366 * holding the struct mutex, yet we want to hold it while
367 * dereferencing the user data.
368 */
369 first_data_page = data_ptr / PAGE_SIZE;
370 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
371 num_pages = last_data_page - first_data_page + 1;
372
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700373 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
Eric Anholteb014592009-03-10 11:44:52 -0700374 if (user_pages == NULL)
375 return -ENOMEM;
376
377 down_read(&mm->mmap_sem);
378 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
Eric Anholte5e9ecd2009-04-07 16:01:22 -0700379 num_pages, 1, 0, user_pages, NULL);
Eric Anholteb014592009-03-10 11:44:52 -0700380 up_read(&mm->mmap_sem);
381 if (pinned_pages < num_pages) {
382 ret = -EFAULT;
383 goto fail_put_user_pages;
384 }
385
Eric Anholt280b7132009-03-12 16:56:27 -0700386 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
387
Eric Anholteb014592009-03-10 11:44:52 -0700388 mutex_lock(&dev->struct_mutex);
389
Chris Wilson07f73f62009-09-14 16:50:30 +0100390 ret = i915_gem_object_get_pages_or_evict(obj);
391 if (ret)
Eric Anholteb014592009-03-10 11:44:52 -0700392 goto fail_unlock;
393
394 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
395 args->size);
396 if (ret != 0)
397 goto fail_put_pages;
398
Daniel Vetter23010e42010-03-08 13:35:02 +0100399 obj_priv = to_intel_bo(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700400 offset = args->offset;
401
402 while (remain > 0) {
403 /* Operation in this page
404 *
405 * shmem_page_index = page number within shmem file
406 * shmem_page_offset = offset within page in shmem file
407 * data_page_index = page number in get_user_pages return
408 * data_page_offset = offset with data_page_index page.
409 * page_length = bytes to copy for this page
410 */
411 shmem_page_index = offset / PAGE_SIZE;
412 shmem_page_offset = offset & ~PAGE_MASK;
413 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
414 data_page_offset = data_ptr & ~PAGE_MASK;
415
416 page_length = remain;
417 if ((shmem_page_offset + page_length) > PAGE_SIZE)
418 page_length = PAGE_SIZE - shmem_page_offset;
419 if ((data_page_offset + page_length) > PAGE_SIZE)
420 page_length = PAGE_SIZE - data_page_offset;
421
Eric Anholt280b7132009-03-12 16:56:27 -0700422 if (do_bit17_swizzling) {
Chris Wilson99a03df2010-05-27 14:15:34 +0100423 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
Eric Anholt280b7132009-03-12 16:56:27 -0700424 shmem_page_offset,
Chris Wilson99a03df2010-05-27 14:15:34 +0100425 user_pages[data_page_index],
426 data_page_offset,
427 page_length,
428 1);
429 } else {
430 slow_shmem_copy(user_pages[data_page_index],
431 data_page_offset,
432 obj_priv->pages[shmem_page_index],
433 shmem_page_offset,
434 page_length);
Eric Anholt280b7132009-03-12 16:56:27 -0700435 }
Eric Anholteb014592009-03-10 11:44:52 -0700436
437 remain -= page_length;
438 data_ptr += page_length;
439 offset += page_length;
440 }
441
442fail_put_pages:
443 i915_gem_object_put_pages(obj);
444fail_unlock:
445 mutex_unlock(&dev->struct_mutex);
446fail_put_user_pages:
447 for (i = 0; i < pinned_pages; i++) {
448 SetPageDirty(user_pages[i]);
449 page_cache_release(user_pages[i]);
450 }
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700451 drm_free_large(user_pages);
Eric Anholteb014592009-03-10 11:44:52 -0700452
453 return ret;
454}
455
Eric Anholt673a3942008-07-30 12:06:12 -0700456/**
457 * Reads data from the object referenced by handle.
458 *
459 * On error, the contents of *data are undefined.
460 */
461int
462i915_gem_pread_ioctl(struct drm_device *dev, void *data,
463 struct drm_file *file_priv)
464{
465 struct drm_i915_gem_pread *args = data;
466 struct drm_gem_object *obj;
467 struct drm_i915_gem_object *obj_priv;
Eric Anholt673a3942008-07-30 12:06:12 -0700468 int ret;
469
470 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
471 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +0100472 return -ENOENT;
Daniel Vetter23010e42010-03-08 13:35:02 +0100473 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700474
475 /* Bounds check source.
476 *
477 * XXX: This could use review for overflow issues...
478 */
479 if (args->offset > obj->size || args->size > obj->size ||
480 args->offset + args->size > obj->size) {
Luca Barbieribc9025b2010-02-09 05:49:12 +0000481 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700482 return -EINVAL;
483 }
484
Eric Anholt280b7132009-03-12 16:56:27 -0700485 if (i915_gem_object_needs_bit17_swizzle(obj)) {
Eric Anholteb014592009-03-10 11:44:52 -0700486 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
Eric Anholt280b7132009-03-12 16:56:27 -0700487 } else {
488 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
489 if (ret != 0)
490 ret = i915_gem_shmem_pread_slow(dev, obj, args,
491 file_priv);
492 }
Eric Anholt673a3942008-07-30 12:06:12 -0700493
Luca Barbieribc9025b2010-02-09 05:49:12 +0000494 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700495
Eric Anholteb014592009-03-10 11:44:52 -0700496 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700497}
498
Keith Packard0839ccb2008-10-30 19:38:48 -0700499/* This is the fast write path which cannot handle
500 * page faults in the source data
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700501 */
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700502
Keith Packard0839ccb2008-10-30 19:38:48 -0700503static inline int
504fast_user_write(struct io_mapping *mapping,
505 loff_t page_base, int page_offset,
506 char __user *user_data,
507 int length)
508{
509 char *vaddr_atomic;
510 unsigned long unwritten;
511
Chris Wilsonfca3ec02010-08-04 14:34:24 +0100512 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
Keith Packard0839ccb2008-10-30 19:38:48 -0700513 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
514 user_data, length);
Chris Wilsonfca3ec02010-08-04 14:34:24 +0100515 io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
Keith Packard0839ccb2008-10-30 19:38:48 -0700516 if (unwritten)
517 return -EFAULT;
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700518 return 0;
Keith Packard0839ccb2008-10-30 19:38:48 -0700519}
520
521/* Here's the write path which can sleep for
522 * page faults
523 */
524
Chris Wilsonab34c222010-05-27 14:15:35 +0100525static inline void
Eric Anholt3de09aa2009-03-09 09:42:23 -0700526slow_kernel_write(struct io_mapping *mapping,
527 loff_t gtt_base, int gtt_offset,
528 struct page *user_page, int user_offset,
529 int length)
Keith Packard0839ccb2008-10-30 19:38:48 -0700530{
Chris Wilsonab34c222010-05-27 14:15:35 +0100531 char __iomem *dst_vaddr;
532 char *src_vaddr;
Keith Packard0839ccb2008-10-30 19:38:48 -0700533
Chris Wilsonab34c222010-05-27 14:15:35 +0100534 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
535 src_vaddr = kmap(user_page);
536
537 memcpy_toio(dst_vaddr + gtt_offset,
538 src_vaddr + user_offset,
539 length);
540
541 kunmap(user_page);
542 io_mapping_unmap(dst_vaddr);
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700543}
544
Eric Anholt40123c12009-03-09 13:42:30 -0700545static inline int
546fast_shmem_write(struct page **pages,
547 loff_t page_base, int page_offset,
548 char __user *data,
549 int length)
550{
551 char __iomem *vaddr;
Dave Airlied0088772009-03-28 20:29:48 -0400552 unsigned long unwritten;
Eric Anholt40123c12009-03-09 13:42:30 -0700553
554 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
555 if (vaddr == NULL)
556 return -ENOMEM;
Dave Airlied0088772009-03-28 20:29:48 -0400557 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
Eric Anholt40123c12009-03-09 13:42:30 -0700558 kunmap_atomic(vaddr, KM_USER0);
559
Dave Airlied0088772009-03-28 20:29:48 -0400560 if (unwritten)
561 return -EFAULT;
Eric Anholt40123c12009-03-09 13:42:30 -0700562 return 0;
563}
564
Eric Anholt3de09aa2009-03-09 09:42:23 -0700565/**
566 * This is the fast pwrite path, where we copy the data directly from the
567 * user into the GTT, uncached.
568 */
Eric Anholt673a3942008-07-30 12:06:12 -0700569static int
Eric Anholt3de09aa2009-03-09 09:42:23 -0700570i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
571 struct drm_i915_gem_pwrite *args,
572 struct drm_file *file_priv)
Eric Anholt673a3942008-07-30 12:06:12 -0700573{
Daniel Vetter23010e42010-03-08 13:35:02 +0100574 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Keith Packard0839ccb2008-10-30 19:38:48 -0700575 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -0700576 ssize_t remain;
Keith Packard0839ccb2008-10-30 19:38:48 -0700577 loff_t offset, page_base;
Eric Anholt673a3942008-07-30 12:06:12 -0700578 char __user *user_data;
Keith Packard0839ccb2008-10-30 19:38:48 -0700579 int page_offset, page_length;
580 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700581
582 user_data = (char __user *) (uintptr_t) args->data_ptr;
583 remain = args->size;
584 if (!access_ok(VERIFY_READ, user_data, remain))
585 return -EFAULT;
586
587
588 mutex_lock(&dev->struct_mutex);
589 ret = i915_gem_object_pin(obj, 0);
590 if (ret) {
591 mutex_unlock(&dev->struct_mutex);
592 return ret;
593 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800594 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
Eric Anholt673a3942008-07-30 12:06:12 -0700595 if (ret)
596 goto fail;
597
Daniel Vetter23010e42010-03-08 13:35:02 +0100598 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700599 offset = obj_priv->gtt_offset + args->offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700600
601 while (remain > 0) {
602 /* Operation in this page
603 *
Keith Packard0839ccb2008-10-30 19:38:48 -0700604 * page_base = page offset within aperture
605 * page_offset = offset within page
606 * page_length = bytes to copy for this page
Eric Anholt673a3942008-07-30 12:06:12 -0700607 */
Keith Packard0839ccb2008-10-30 19:38:48 -0700608 page_base = (offset & ~(PAGE_SIZE-1));
609 page_offset = offset & (PAGE_SIZE-1);
610 page_length = remain;
611 if ((page_offset + remain) > PAGE_SIZE)
612 page_length = PAGE_SIZE - page_offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700613
Keith Packard0839ccb2008-10-30 19:38:48 -0700614 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
615 page_offset, user_data, page_length);
Eric Anholt673a3942008-07-30 12:06:12 -0700616
Keith Packard0839ccb2008-10-30 19:38:48 -0700617 /* If we get a fault while copying data, then (presumably) our
Eric Anholt3de09aa2009-03-09 09:42:23 -0700618 * source page isn't available. Return the error and we'll
619 * retry in the slow path.
Keith Packard0839ccb2008-10-30 19:38:48 -0700620 */
Eric Anholt3de09aa2009-03-09 09:42:23 -0700621 if (ret)
622 goto fail;
Eric Anholt673a3942008-07-30 12:06:12 -0700623
Keith Packard0839ccb2008-10-30 19:38:48 -0700624 remain -= page_length;
625 user_data += page_length;
626 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700627 }
Eric Anholt673a3942008-07-30 12:06:12 -0700628
629fail:
630 i915_gem_object_unpin(obj);
631 mutex_unlock(&dev->struct_mutex);
632
633 return ret;
634}
635
Eric Anholt3de09aa2009-03-09 09:42:23 -0700636/**
637 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
638 * the memory and maps it using kmap_atomic for copying.
639 *
640 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
641 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
642 */
Eric Anholt3043c602008-10-02 12:24:47 -0700643static int
Eric Anholt3de09aa2009-03-09 09:42:23 -0700644i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
645 struct drm_i915_gem_pwrite *args,
646 struct drm_file *file_priv)
Eric Anholt673a3942008-07-30 12:06:12 -0700647{
Daniel Vetter23010e42010-03-08 13:35:02 +0100648 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700649 drm_i915_private_t *dev_priv = dev->dev_private;
650 ssize_t remain;
651 loff_t gtt_page_base, offset;
652 loff_t first_data_page, last_data_page, num_pages;
653 loff_t pinned_pages, i;
654 struct page **user_pages;
655 struct mm_struct *mm = current->mm;
656 int gtt_page_offset, data_page_offset, data_page_index, page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700657 int ret;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700658 uint64_t data_ptr = args->data_ptr;
659
660 remain = args->size;
661
662 /* Pin the user pages containing the data. We can't fault while
663 * holding the struct mutex, and all of the pwrite implementations
664 * want to hold it while dereferencing the user data.
665 */
666 first_data_page = data_ptr / PAGE_SIZE;
667 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
668 num_pages = last_data_page - first_data_page + 1;
669
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700670 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
Eric Anholt3de09aa2009-03-09 09:42:23 -0700671 if (user_pages == NULL)
672 return -ENOMEM;
673
674 down_read(&mm->mmap_sem);
675 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
676 num_pages, 0, 0, user_pages, NULL);
677 up_read(&mm->mmap_sem);
678 if (pinned_pages < num_pages) {
679 ret = -EFAULT;
680 goto out_unpin_pages;
681 }
682
683 mutex_lock(&dev->struct_mutex);
684 ret = i915_gem_object_pin(obj, 0);
685 if (ret)
686 goto out_unlock;
687
688 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
689 if (ret)
690 goto out_unpin_object;
691
Daniel Vetter23010e42010-03-08 13:35:02 +0100692 obj_priv = to_intel_bo(obj);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700693 offset = obj_priv->gtt_offset + args->offset;
694
695 while (remain > 0) {
696 /* Operation in this page
697 *
698 * gtt_page_base = page offset within aperture
699 * gtt_page_offset = offset within page in aperture
700 * data_page_index = page number in get_user_pages return
701 * data_page_offset = offset with data_page_index page.
702 * page_length = bytes to copy for this page
703 */
704 gtt_page_base = offset & PAGE_MASK;
705 gtt_page_offset = offset & ~PAGE_MASK;
706 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
707 data_page_offset = data_ptr & ~PAGE_MASK;
708
709 page_length = remain;
710 if ((gtt_page_offset + page_length) > PAGE_SIZE)
711 page_length = PAGE_SIZE - gtt_page_offset;
712 if ((data_page_offset + page_length) > PAGE_SIZE)
713 page_length = PAGE_SIZE - data_page_offset;
714
Chris Wilsonab34c222010-05-27 14:15:35 +0100715 slow_kernel_write(dev_priv->mm.gtt_mapping,
716 gtt_page_base, gtt_page_offset,
717 user_pages[data_page_index],
718 data_page_offset,
719 page_length);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700720
721 remain -= page_length;
722 offset += page_length;
723 data_ptr += page_length;
724 }
725
726out_unpin_object:
727 i915_gem_object_unpin(obj);
728out_unlock:
729 mutex_unlock(&dev->struct_mutex);
730out_unpin_pages:
731 for (i = 0; i < pinned_pages; i++)
732 page_cache_release(user_pages[i]);
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700733 drm_free_large(user_pages);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700734
735 return ret;
736}
737
Eric Anholt40123c12009-03-09 13:42:30 -0700738/**
739 * This is the fast shmem pwrite path, which attempts to directly
740 * copy_from_user into the kmapped pages backing the object.
741 */
Eric Anholt673a3942008-07-30 12:06:12 -0700742static int
Eric Anholt40123c12009-03-09 13:42:30 -0700743i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
744 struct drm_i915_gem_pwrite *args,
745 struct drm_file *file_priv)
Eric Anholt673a3942008-07-30 12:06:12 -0700746{
Daniel Vetter23010e42010-03-08 13:35:02 +0100747 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt40123c12009-03-09 13:42:30 -0700748 ssize_t remain;
749 loff_t offset, page_base;
750 char __user *user_data;
751 int page_offset, page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700752 int ret;
Eric Anholt40123c12009-03-09 13:42:30 -0700753
754 user_data = (char __user *) (uintptr_t) args->data_ptr;
755 remain = args->size;
Eric Anholt673a3942008-07-30 12:06:12 -0700756
757 mutex_lock(&dev->struct_mutex);
758
Chris Wilson4bdadb92010-01-27 13:36:32 +0000759 ret = i915_gem_object_get_pages(obj, 0);
Eric Anholt40123c12009-03-09 13:42:30 -0700760 if (ret != 0)
761 goto fail_unlock;
762
Eric Anholte47c68e2008-11-14 13:35:19 -0800763 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
Eric Anholt40123c12009-03-09 13:42:30 -0700764 if (ret != 0)
765 goto fail_put_pages;
Eric Anholt673a3942008-07-30 12:06:12 -0700766
Daniel Vetter23010e42010-03-08 13:35:02 +0100767 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700768 offset = args->offset;
Eric Anholt40123c12009-03-09 13:42:30 -0700769 obj_priv->dirty = 1;
Eric Anholt673a3942008-07-30 12:06:12 -0700770
Eric Anholt40123c12009-03-09 13:42:30 -0700771 while (remain > 0) {
772 /* Operation in this page
773 *
774 * page_base = page offset within aperture
775 * page_offset = offset within page
776 * page_length = bytes to copy for this page
777 */
778 page_base = (offset & ~(PAGE_SIZE-1));
779 page_offset = offset & (PAGE_SIZE-1);
780 page_length = remain;
781 if ((page_offset + remain) > PAGE_SIZE)
782 page_length = PAGE_SIZE - page_offset;
783
784 ret = fast_shmem_write(obj_priv->pages,
785 page_base, page_offset,
786 user_data, page_length);
787 if (ret)
788 goto fail_put_pages;
789
790 remain -= page_length;
791 user_data += page_length;
792 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700793 }
794
Eric Anholt40123c12009-03-09 13:42:30 -0700795fail_put_pages:
796 i915_gem_object_put_pages(obj);
797fail_unlock:
Eric Anholt673a3942008-07-30 12:06:12 -0700798 mutex_unlock(&dev->struct_mutex);
799
Eric Anholt40123c12009-03-09 13:42:30 -0700800 return ret;
801}
802
803/**
804 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
805 * the memory and maps it using kmap_atomic for copying.
806 *
807 * This avoids taking mmap_sem for faulting on the user's address while the
808 * struct_mutex is held.
809 */
810static int
811i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
812 struct drm_i915_gem_pwrite *args,
813 struct drm_file *file_priv)
814{
Daniel Vetter23010e42010-03-08 13:35:02 +0100815 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt40123c12009-03-09 13:42:30 -0700816 struct mm_struct *mm = current->mm;
817 struct page **user_pages;
818 ssize_t remain;
819 loff_t offset, pinned_pages, i;
820 loff_t first_data_page, last_data_page, num_pages;
821 int shmem_page_index, shmem_page_offset;
822 int data_page_index, data_page_offset;
823 int page_length;
824 int ret;
825 uint64_t data_ptr = args->data_ptr;
Eric Anholt280b7132009-03-12 16:56:27 -0700826 int do_bit17_swizzling;
Eric Anholt40123c12009-03-09 13:42:30 -0700827
828 remain = args->size;
829
830 /* Pin the user pages containing the data. We can't fault while
831 * holding the struct mutex, and all of the pwrite implementations
832 * want to hold it while dereferencing the user data.
833 */
834 first_data_page = data_ptr / PAGE_SIZE;
835 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
836 num_pages = last_data_page - first_data_page + 1;
837
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700838 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
Eric Anholt40123c12009-03-09 13:42:30 -0700839 if (user_pages == NULL)
840 return -ENOMEM;
841
842 down_read(&mm->mmap_sem);
843 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
844 num_pages, 0, 0, user_pages, NULL);
845 up_read(&mm->mmap_sem);
846 if (pinned_pages < num_pages) {
847 ret = -EFAULT;
848 goto fail_put_user_pages;
849 }
850
Eric Anholt280b7132009-03-12 16:56:27 -0700851 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
852
Eric Anholt40123c12009-03-09 13:42:30 -0700853 mutex_lock(&dev->struct_mutex);
854
Chris Wilson07f73f62009-09-14 16:50:30 +0100855 ret = i915_gem_object_get_pages_or_evict(obj);
856 if (ret)
Eric Anholt40123c12009-03-09 13:42:30 -0700857 goto fail_unlock;
858
859 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
860 if (ret != 0)
861 goto fail_put_pages;
862
Daniel Vetter23010e42010-03-08 13:35:02 +0100863 obj_priv = to_intel_bo(obj);
Eric Anholt40123c12009-03-09 13:42:30 -0700864 offset = args->offset;
865 obj_priv->dirty = 1;
866
867 while (remain > 0) {
868 /* Operation in this page
869 *
870 * shmem_page_index = page number within shmem file
871 * shmem_page_offset = offset within page in shmem file
872 * data_page_index = page number in get_user_pages return
873 * data_page_offset = offset with data_page_index page.
874 * page_length = bytes to copy for this page
875 */
876 shmem_page_index = offset / PAGE_SIZE;
877 shmem_page_offset = offset & ~PAGE_MASK;
878 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
879 data_page_offset = data_ptr & ~PAGE_MASK;
880
881 page_length = remain;
882 if ((shmem_page_offset + page_length) > PAGE_SIZE)
883 page_length = PAGE_SIZE - shmem_page_offset;
884 if ((data_page_offset + page_length) > PAGE_SIZE)
885 page_length = PAGE_SIZE - data_page_offset;
886
Eric Anholt280b7132009-03-12 16:56:27 -0700887 if (do_bit17_swizzling) {
Chris Wilson99a03df2010-05-27 14:15:34 +0100888 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
Eric Anholt280b7132009-03-12 16:56:27 -0700889 shmem_page_offset,
890 user_pages[data_page_index],
891 data_page_offset,
Chris Wilson99a03df2010-05-27 14:15:34 +0100892 page_length,
893 0);
894 } else {
895 slow_shmem_copy(obj_priv->pages[shmem_page_index],
896 shmem_page_offset,
897 user_pages[data_page_index],
898 data_page_offset,
899 page_length);
Eric Anholt280b7132009-03-12 16:56:27 -0700900 }
Eric Anholt40123c12009-03-09 13:42:30 -0700901
902 remain -= page_length;
903 data_ptr += page_length;
904 offset += page_length;
905 }
906
907fail_put_pages:
908 i915_gem_object_put_pages(obj);
909fail_unlock:
910 mutex_unlock(&dev->struct_mutex);
911fail_put_user_pages:
912 for (i = 0; i < pinned_pages; i++)
913 page_cache_release(user_pages[i]);
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700914 drm_free_large(user_pages);
Eric Anholt40123c12009-03-09 13:42:30 -0700915
916 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700917}
918
919/**
920 * Writes data to the object referenced by handle.
921 *
922 * On error, the contents of the buffer that were to be modified are undefined.
923 */
924int
925i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
926 struct drm_file *file_priv)
927{
928 struct drm_i915_gem_pwrite *args = data;
929 struct drm_gem_object *obj;
930 struct drm_i915_gem_object *obj_priv;
931 int ret = 0;
932
933 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
934 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +0100935 return -ENOENT;
Daniel Vetter23010e42010-03-08 13:35:02 +0100936 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700937
938 /* Bounds check destination.
939 *
940 * XXX: This could use review for overflow issues...
941 */
942 if (args->offset > obj->size || args->size > obj->size ||
943 args->offset + args->size > obj->size) {
Luca Barbieribc9025b2010-02-09 05:49:12 +0000944 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700945 return -EINVAL;
946 }
947
948 /* We can only do the GTT pwrite on untiled buffers, as otherwise
949 * it would end up going through the fenced access, and we'll get
950 * different detiling behavior between reading and writing.
951 * pread/pwrite currently are reading and writing from the CPU
952 * perspective, requiring manual detiling by the client.
953 */
Dave Airlie71acb5e2008-12-30 20:31:46 +1000954 if (obj_priv->phys_obj)
955 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
956 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
Chris Wilson9b8c4a02010-05-27 14:21:01 +0100957 dev->gtt_total != 0 &&
958 obj->write_domain != I915_GEM_DOMAIN_CPU) {
Eric Anholt3de09aa2009-03-09 09:42:23 -0700959 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
960 if (ret == -EFAULT) {
961 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
962 file_priv);
963 }
Eric Anholt280b7132009-03-12 16:56:27 -0700964 } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
965 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
Eric Anholt40123c12009-03-09 13:42:30 -0700966 } else {
967 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
968 if (ret == -EFAULT) {
969 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
970 file_priv);
971 }
972 }
Eric Anholt673a3942008-07-30 12:06:12 -0700973
974#if WATCH_PWRITE
975 if (ret)
976 DRM_INFO("pwrite failed %d\n", ret);
977#endif
978
Luca Barbieribc9025b2010-02-09 05:49:12 +0000979 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700980
981 return ret;
982}
983
984/**
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800985 * Called when user space prepares to use an object with the CPU, either
986 * through the mmap ioctl's mapping or a GTT mapping.
Eric Anholt673a3942008-07-30 12:06:12 -0700987 */
988int
989i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
990 struct drm_file *file_priv)
991{
Eric Anholta09ba7f2009-08-29 12:49:51 -0700992 struct drm_i915_private *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -0700993 struct drm_i915_gem_set_domain *args = data;
994 struct drm_gem_object *obj;
Jesse Barnes652c3932009-08-17 13:31:43 -0700995 struct drm_i915_gem_object *obj_priv;
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800996 uint32_t read_domains = args->read_domains;
997 uint32_t write_domain = args->write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -0700998 int ret;
999
1000 if (!(dev->driver->driver_features & DRIVER_GEM))
1001 return -ENODEV;
1002
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001003 /* Only handle setting domains to types used by the CPU. */
Chris Wilson21d509e2009-06-06 09:46:02 +01001004 if (write_domain & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001005 return -EINVAL;
1006
Chris Wilson21d509e2009-06-06 09:46:02 +01001007 if (read_domains & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001008 return -EINVAL;
1009
1010 /* Having something in the write domain implies it's in the read
1011 * domain, and only that read domain. Enforce that in the request.
1012 */
1013 if (write_domain != 0 && read_domains != write_domain)
1014 return -EINVAL;
1015
Eric Anholt673a3942008-07-30 12:06:12 -07001016 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1017 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +01001018 return -ENOENT;
Daniel Vetter23010e42010-03-08 13:35:02 +01001019 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001020
1021 mutex_lock(&dev->struct_mutex);
Jesse Barnes652c3932009-08-17 13:31:43 -07001022
1023 intel_mark_busy(dev, obj);
1024
Eric Anholt673a3942008-07-30 12:06:12 -07001025#if WATCH_BUF
Krzysztof Halasacfd43c02009-06-20 00:31:28 +02001026 DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001027 obj, obj->size, read_domains, write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07001028#endif
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001029 if (read_domains & I915_GEM_DOMAIN_GTT) {
1030 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
Eric Anholt02354392008-11-26 13:58:13 -08001031
Eric Anholta09ba7f2009-08-29 12:49:51 -07001032 /* Update the LRU on the fence for the CPU access that's
1033 * about to occur.
1034 */
1035 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
Daniel Vetter007cc8a2010-04-28 11:02:31 +02001036 struct drm_i915_fence_reg *reg =
1037 &dev_priv->fence_regs[obj_priv->fence_reg];
1038 list_move_tail(&reg->lru_list,
Eric Anholta09ba7f2009-08-29 12:49:51 -07001039 &dev_priv->mm.fence_list);
1040 }
1041
Eric Anholt02354392008-11-26 13:58:13 -08001042 /* Silently promote "you're not bound, there was nothing to do"
1043 * to success, since the client was just asking us to
1044 * make sure everything was done.
1045 */
1046 if (ret == -EINVAL)
1047 ret = 0;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001048 } else {
Eric Anholte47c68e2008-11-14 13:35:19 -08001049 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001050 }
1051
Chris Wilson7d1c4802010-08-07 21:45:03 +01001052
1053 /* Maintain LRU order of "inactive" objects */
1054 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1055 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1056
Eric Anholt673a3942008-07-30 12:06:12 -07001057 drm_gem_object_unreference(obj);
1058 mutex_unlock(&dev->struct_mutex);
1059 return ret;
1060}
1061
1062/**
1063 * Called when user space has done writes to this buffer
1064 */
1065int
1066i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1067 struct drm_file *file_priv)
1068{
1069 struct drm_i915_gem_sw_finish *args = data;
1070 struct drm_gem_object *obj;
1071 struct drm_i915_gem_object *obj_priv;
1072 int ret = 0;
1073
1074 if (!(dev->driver->driver_features & DRIVER_GEM))
1075 return -ENODEV;
1076
1077 mutex_lock(&dev->struct_mutex);
1078 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1079 if (obj == NULL) {
1080 mutex_unlock(&dev->struct_mutex);
Chris Wilsonbf79cb92010-08-04 14:19:46 +01001081 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07001082 }
1083
1084#if WATCH_BUF
Krzysztof Halasacfd43c02009-06-20 00:31:28 +02001085 DRM_INFO("%s: sw_finish %d (%p %zd)\n",
Eric Anholt673a3942008-07-30 12:06:12 -07001086 __func__, args->handle, obj, obj->size);
1087#endif
Daniel Vetter23010e42010-03-08 13:35:02 +01001088 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001089
1090 /* Pinned buffers may be scanout, so flush the cache */
Eric Anholte47c68e2008-11-14 13:35:19 -08001091 if (obj_priv->pin_count)
1092 i915_gem_object_flush_cpu_write_domain(obj);
1093
Eric Anholt673a3942008-07-30 12:06:12 -07001094 drm_gem_object_unreference(obj);
1095 mutex_unlock(&dev->struct_mutex);
1096 return ret;
1097}
1098
1099/**
1100 * Maps the contents of an object, returning the address it is mapped
1101 * into.
1102 *
1103 * While the mapping holds a reference on the contents of the object, it doesn't
1104 * imply a ref on the object itself.
1105 */
1106int
1107i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1108 struct drm_file *file_priv)
1109{
1110 struct drm_i915_gem_mmap *args = data;
1111 struct drm_gem_object *obj;
1112 loff_t offset;
1113 unsigned long addr;
1114
1115 if (!(dev->driver->driver_features & DRIVER_GEM))
1116 return -ENODEV;
1117
1118 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1119 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +01001120 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07001121
1122 offset = args->offset;
1123
1124 down_write(&current->mm->mmap_sem);
1125 addr = do_mmap(obj->filp, 0, args->size,
1126 PROT_READ | PROT_WRITE, MAP_SHARED,
1127 args->offset);
1128 up_write(&current->mm->mmap_sem);
Luca Barbieribc9025b2010-02-09 05:49:12 +00001129 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001130 if (IS_ERR((void *)addr))
1131 return addr;
1132
1133 args->addr_ptr = (uint64_t) addr;
1134
1135 return 0;
1136}
1137
Jesse Barnesde151cf2008-11-12 10:03:55 -08001138/**
1139 * i915_gem_fault - fault a page into the GTT
1140 * vma: VMA in question
1141 * vmf: fault info
1142 *
1143 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1144 * from userspace. The fault handler takes care of binding the object to
1145 * the GTT (if needed), allocating and programming a fence register (again,
1146 * only if needed based on whether the old reg is still valid or the object
1147 * is tiled) and inserting a new PTE into the faulting process.
1148 *
1149 * Note that the faulting process may involve evicting existing objects
1150 * from the GTT and/or fence registers to make room. So performance may
1151 * suffer if the GTT working set is large or there are few fence registers
1152 * left.
1153 */
1154int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1155{
1156 struct drm_gem_object *obj = vma->vm_private_data;
1157 struct drm_device *dev = obj->dev;
Chris Wilson7d1c4802010-08-07 21:45:03 +01001158 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001159 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001160 pgoff_t page_offset;
1161 unsigned long pfn;
1162 int ret = 0;
Jesse Barnes0f973f22009-01-26 17:10:45 -08001163 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001164
1165 /* We don't use vmf->pgoff since that has the fake offset */
1166 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1167 PAGE_SHIFT;
1168
1169 /* Now bind it into the GTT if needed */
1170 mutex_lock(&dev->struct_mutex);
1171 if (!obj_priv->gtt_space) {
Chris Wilsone67b8ce2009-09-14 16:50:26 +01001172 ret = i915_gem_object_bind_to_gtt(obj, 0);
Chris Wilsonc7150892009-09-23 00:43:56 +01001173 if (ret)
1174 goto unlock;
Kristian Høgsberg07f4f3e2009-05-27 14:37:28 -04001175
Jesse Barnesde151cf2008-11-12 10:03:55 -08001176 ret = i915_gem_object_set_to_gtt_domain(obj, write);
Chris Wilsonc7150892009-09-23 00:43:56 +01001177 if (ret)
1178 goto unlock;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001179 }
1180
1181 /* Need a new fence register? */
Eric Anholta09ba7f2009-08-29 12:49:51 -07001182 if (obj_priv->tiling_mode != I915_TILING_NONE) {
Chris Wilson8c4b8c32009-06-17 22:08:52 +01001183 ret = i915_gem_object_get_fence_reg(obj);
Chris Wilsonc7150892009-09-23 00:43:56 +01001184 if (ret)
1185 goto unlock;
Eric Anholtd9ddcb92009-01-27 10:33:49 -08001186 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08001187
Chris Wilson7d1c4802010-08-07 21:45:03 +01001188 if (i915_gem_object_is_inactive(obj_priv))
1189 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1190
Jesse Barnesde151cf2008-11-12 10:03:55 -08001191 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1192 page_offset;
1193
1194 /* Finally, remap it using the new GTT offset */
1195 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
Chris Wilsonc7150892009-09-23 00:43:56 +01001196unlock:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001197 mutex_unlock(&dev->struct_mutex);
1198
1199 switch (ret) {
Chris Wilsonc7150892009-09-23 00:43:56 +01001200 case 0:
1201 case -ERESTARTSYS:
1202 return VM_FAULT_NOPAGE;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001203 case -ENOMEM:
1204 case -EAGAIN:
1205 return VM_FAULT_OOM;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001206 default:
Chris Wilsonc7150892009-09-23 00:43:56 +01001207 return VM_FAULT_SIGBUS;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001208 }
1209}
1210
1211/**
1212 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1213 * @obj: obj in question
1214 *
1215 * GEM memory mapping works by handing back to userspace a fake mmap offset
1216 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1217 * up the object based on the offset and sets up the various memory mapping
1218 * structures.
1219 *
1220 * This routine allocates and attaches a fake offset for @obj.
1221 */
1222static int
1223i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1224{
1225 struct drm_device *dev = obj->dev;
1226 struct drm_gem_mm *mm = dev->mm_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001227 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001228 struct drm_map_list *list;
Benjamin Herrenschmidtf77d3902009-02-02 16:55:46 +11001229 struct drm_local_map *map;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001230 int ret = 0;
1231
1232 /* Set the object up for mmap'ing */
1233 list = &obj->map_list;
Eric Anholt9a298b22009-03-24 12:23:04 -07001234 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001235 if (!list->map)
1236 return -ENOMEM;
1237
1238 map = list->map;
1239 map->type = _DRM_GEM;
1240 map->size = obj->size;
1241 map->handle = obj;
1242
1243 /* Get a DRM GEM mmap offset allocated... */
1244 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1245 obj->size / PAGE_SIZE, 0, 0);
1246 if (!list->file_offset_node) {
1247 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1248 ret = -ENOMEM;
1249 goto out_free_list;
1250 }
1251
1252 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1253 obj->size / PAGE_SIZE, 0);
1254 if (!list->file_offset_node) {
1255 ret = -ENOMEM;
1256 goto out_free_list;
1257 }
1258
1259 list->hash.key = list->file_offset_node->start;
1260 if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1261 DRM_ERROR("failed to add to map hash\n");
Chris Wilson5618ca62009-12-02 15:15:30 +00001262 ret = -ENOMEM;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001263 goto out_free_mm;
1264 }
1265
1266 /* By now we should be all set, any drm_mmap request on the offset
1267 * below will get to our mmap & fault handler */
1268 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1269
1270 return 0;
1271
1272out_free_mm:
1273 drm_mm_put_block(list->file_offset_node);
1274out_free_list:
Eric Anholt9a298b22009-03-24 12:23:04 -07001275 kfree(list->map);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001276
1277 return ret;
1278}
1279
Chris Wilson901782b2009-07-10 08:18:50 +01001280/**
1281 * i915_gem_release_mmap - remove physical page mappings
1282 * @obj: obj in question
1283 *
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02001284 * Preserve the reservation of the mmapping with the DRM core code, but
Chris Wilson901782b2009-07-10 08:18:50 +01001285 * relinquish ownership of the pages back to the system.
1286 *
1287 * It is vital that we remove the page mapping if we have mapped a tiled
1288 * object through the GTT and then lose the fence register due to
1289 * resource pressure. Similarly if the object has been moved out of the
1290 * aperture, than pages mapped into userspace must be revoked. Removing the
1291 * mapping will then trigger a page fault on the next user access, allowing
1292 * fixup by i915_gem_fault().
1293 */
Eric Anholtd05ca302009-07-10 13:02:26 -07001294void
Chris Wilson901782b2009-07-10 08:18:50 +01001295i915_gem_release_mmap(struct drm_gem_object *obj)
1296{
1297 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001298 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson901782b2009-07-10 08:18:50 +01001299
1300 if (dev->dev_mapping)
1301 unmap_mapping_range(dev->dev_mapping,
1302 obj_priv->mmap_offset, obj->size, 1);
1303}
1304
Jesse Barnesab00b3e2009-02-11 14:01:46 -08001305static void
1306i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1307{
1308 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001309 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesab00b3e2009-02-11 14:01:46 -08001310 struct drm_gem_mm *mm = dev->mm_private;
1311 struct drm_map_list *list;
1312
1313 list = &obj->map_list;
1314 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1315
1316 if (list->file_offset_node) {
1317 drm_mm_put_block(list->file_offset_node);
1318 list->file_offset_node = NULL;
1319 }
1320
1321 if (list->map) {
Eric Anholt9a298b22009-03-24 12:23:04 -07001322 kfree(list->map);
Jesse Barnesab00b3e2009-02-11 14:01:46 -08001323 list->map = NULL;
1324 }
1325
1326 obj_priv->mmap_offset = 0;
1327}
1328
Jesse Barnesde151cf2008-11-12 10:03:55 -08001329/**
1330 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1331 * @obj: object to check
1332 *
1333 * Return the required GTT alignment for an object, taking into account
1334 * potential fence register mapping if needed.
1335 */
1336static uint32_t
1337i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1338{
1339 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001340 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001341 int start, i;
1342
1343 /*
1344 * Minimum alignment is 4k (GTT page size), but might be greater
1345 * if a fence register is needed for the object.
1346 */
1347 if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1348 return 4096;
1349
1350 /*
1351 * Previous chips need to be aligned to the size of the smallest
1352 * fence register that can contain the object.
1353 */
1354 if (IS_I9XX(dev))
1355 start = 1024*1024;
1356 else
1357 start = 512*1024;
1358
1359 for (i = start; i < obj->size; i <<= 1)
1360 ;
1361
1362 return i;
1363}
1364
1365/**
1366 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1367 * @dev: DRM device
1368 * @data: GTT mapping ioctl data
1369 * @file_priv: GEM object info
1370 *
1371 * Simply returns the fake offset to userspace so it can mmap it.
1372 * The mmap call will end up in drm_gem_mmap(), which will set things
1373 * up so we can get faults in the handler above.
1374 *
1375 * The fault handler will take care of binding the object into the GTT
1376 * (since it may have been evicted to make room for something), allocating
1377 * a fence register, and mapping the appropriate aperture address into
1378 * userspace.
1379 */
1380int
1381i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1382 struct drm_file *file_priv)
1383{
1384 struct drm_i915_gem_mmap_gtt *args = data;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001385 struct drm_gem_object *obj;
1386 struct drm_i915_gem_object *obj_priv;
1387 int ret;
1388
1389 if (!(dev->driver->driver_features & DRIVER_GEM))
1390 return -ENODEV;
1391
1392 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1393 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +01001394 return -ENOENT;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001395
1396 mutex_lock(&dev->struct_mutex);
1397
Daniel Vetter23010e42010-03-08 13:35:02 +01001398 obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001399
Chris Wilsonab182822009-09-22 18:46:17 +01001400 if (obj_priv->madv != I915_MADV_WILLNEED) {
1401 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1402 drm_gem_object_unreference(obj);
1403 mutex_unlock(&dev->struct_mutex);
1404 return -EINVAL;
1405 }
1406
1407
Jesse Barnesde151cf2008-11-12 10:03:55 -08001408 if (!obj_priv->mmap_offset) {
1409 ret = i915_gem_create_mmap_offset(obj);
Chris Wilson13af1062009-02-11 14:26:31 +00001410 if (ret) {
1411 drm_gem_object_unreference(obj);
1412 mutex_unlock(&dev->struct_mutex);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001413 return ret;
Chris Wilson13af1062009-02-11 14:26:31 +00001414 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08001415 }
1416
1417 args->offset = obj_priv->mmap_offset;
1418
Jesse Barnesde151cf2008-11-12 10:03:55 -08001419 /*
1420 * Pull it into the GTT so that we have a page list (makes the
1421 * initial fault faster and any subsequent flushing possible).
1422 */
1423 if (!obj_priv->agp_mem) {
Chris Wilsone67b8ce2009-09-14 16:50:26 +01001424 ret = i915_gem_object_bind_to_gtt(obj, 0);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001425 if (ret) {
1426 drm_gem_object_unreference(obj);
1427 mutex_unlock(&dev->struct_mutex);
1428 return ret;
1429 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08001430 }
1431
1432 drm_gem_object_unreference(obj);
1433 mutex_unlock(&dev->struct_mutex);
1434
1435 return 0;
1436}
1437
Ben Gamari6911a9b2009-04-02 11:24:54 -07001438void
Eric Anholt856fa192009-03-19 14:10:50 -07001439i915_gem_object_put_pages(struct drm_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07001440{
Daniel Vetter23010e42010-03-08 13:35:02 +01001441 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001442 int page_count = obj->size / PAGE_SIZE;
1443 int i;
1444
Eric Anholt856fa192009-03-19 14:10:50 -07001445 BUG_ON(obj_priv->pages_refcount == 0);
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001446 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
Eric Anholt856fa192009-03-19 14:10:50 -07001447
1448 if (--obj_priv->pages_refcount != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07001449 return;
1450
Eric Anholt280b7132009-03-12 16:56:27 -07001451 if (obj_priv->tiling_mode != I915_TILING_NONE)
1452 i915_gem_object_save_bit_17_swizzle(obj);
1453
Chris Wilson3ef94da2009-09-14 16:50:29 +01001454 if (obj_priv->madv == I915_MADV_DONTNEED)
Chris Wilson13a05fd2009-09-20 23:03:19 +01001455 obj_priv->dirty = 0;
Chris Wilson3ef94da2009-09-14 16:50:29 +01001456
1457 for (i = 0; i < page_count; i++) {
Chris Wilson3ef94da2009-09-14 16:50:29 +01001458 if (obj_priv->dirty)
1459 set_page_dirty(obj_priv->pages[i]);
1460
1461 if (obj_priv->madv == I915_MADV_WILLNEED)
Eric Anholt856fa192009-03-19 14:10:50 -07001462 mark_page_accessed(obj_priv->pages[i]);
Chris Wilson3ef94da2009-09-14 16:50:29 +01001463
1464 page_cache_release(obj_priv->pages[i]);
1465 }
Eric Anholt673a3942008-07-30 12:06:12 -07001466 obj_priv->dirty = 0;
1467
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07001468 drm_free_large(obj_priv->pages);
Eric Anholt856fa192009-03-19 14:10:50 -07001469 obj_priv->pages = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07001470}
1471
Daniel Vettere35a41d2010-02-11 22:13:59 +01001472static uint32_t
Daniel Vettera6910432010-02-02 17:08:37 +01001473i915_gem_next_request_seqno(struct drm_device *dev,
1474 struct intel_ring_buffer *ring)
Daniel Vettere35a41d2010-02-11 22:13:59 +01001475{
1476 drm_i915_private_t *dev_priv = dev->dev_private;
1477
Daniel Vettera6910432010-02-02 17:08:37 +01001478 ring->outstanding_lazy_request = true;
1479
Daniel Vettere35a41d2010-02-11 22:13:59 +01001480 return dev_priv->next_seqno;
1481}
1482
Eric Anholt673a3942008-07-30 12:06:12 -07001483static void
Zou Nan hai852835f2010-05-21 09:08:56 +08001484i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno,
1485 struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001486{
1487 struct drm_device *dev = obj->dev;
1488 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001489 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Zou Nan hai852835f2010-05-21 09:08:56 +08001490 BUG_ON(ring == NULL);
1491 obj_priv->ring = ring;
Eric Anholt673a3942008-07-30 12:06:12 -07001492
1493 /* Add a reference if we're newly entering the active list. */
1494 if (!obj_priv->active) {
1495 drm_gem_object_reference(obj);
1496 obj_priv->active = 1;
1497 }
Daniel Vettere35a41d2010-02-11 22:13:59 +01001498
1499 /* Take the seqno of the next request if none is given */
1500 if (seqno == 0)
Daniel Vettera6910432010-02-02 17:08:37 +01001501 seqno = i915_gem_next_request_seqno(dev, ring);
Daniel Vettere35a41d2010-02-11 22:13:59 +01001502
Eric Anholt673a3942008-07-30 12:06:12 -07001503 /* Move from whatever list we were on to the tail of execution. */
Carl Worth5e118f42009-03-20 11:54:25 -07001504 spin_lock(&dev_priv->mm.active_list_lock);
Zou Nan hai852835f2010-05-21 09:08:56 +08001505 list_move_tail(&obj_priv->list, &ring->active_list);
Carl Worth5e118f42009-03-20 11:54:25 -07001506 spin_unlock(&dev_priv->mm.active_list_lock);
Eric Anholtce44b0e2008-11-06 16:00:31 -08001507 obj_priv->last_rendering_seqno = seqno;
Eric Anholt673a3942008-07-30 12:06:12 -07001508}
1509
Eric Anholtce44b0e2008-11-06 16:00:31 -08001510static void
1511i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1512{
1513 struct drm_device *dev = obj->dev;
1514 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001515 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholtce44b0e2008-11-06 16:00:31 -08001516
1517 BUG_ON(!obj_priv->active);
1518 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1519 obj_priv->last_rendering_seqno = 0;
1520}
Eric Anholt673a3942008-07-30 12:06:12 -07001521
Chris Wilson963b4832009-09-20 23:03:54 +01001522/* Immediately discard the backing storage */
1523static void
1524i915_gem_object_truncate(struct drm_gem_object *obj)
1525{
Daniel Vetter23010e42010-03-08 13:35:02 +01001526 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001527 struct inode *inode;
Chris Wilson963b4832009-09-20 23:03:54 +01001528
Chris Wilsonae9fed62010-08-07 11:01:30 +01001529 /* Our goal here is to return as much of the memory as
1530 * is possible back to the system as we are called from OOM.
1531 * To do this we must instruct the shmfs to drop all of its
1532 * backing pages, *now*. Here we mirror the actions taken
1533 * when by shmem_delete_inode() to release the backing store.
1534 */
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001535 inode = obj->filp->f_path.dentry->d_inode;
Chris Wilsonae9fed62010-08-07 11:01:30 +01001536 truncate_inode_pages(inode->i_mapping, 0);
1537 if (inode->i_op->truncate_range)
1538 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001539
1540 obj_priv->madv = __I915_MADV_PURGED;
Chris Wilson963b4832009-09-20 23:03:54 +01001541}
1542
1543static inline int
1544i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1545{
1546 return obj_priv->madv == I915_MADV_DONTNEED;
1547}
1548
Eric Anholt673a3942008-07-30 12:06:12 -07001549static void
1550i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1551{
1552 struct drm_device *dev = obj->dev;
1553 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001554 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001555
1556 i915_verify_inactive(dev, __FILE__, __LINE__);
1557 if (obj_priv->pin_count != 0)
1558 list_del_init(&obj_priv->list);
1559 else
1560 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1561
Daniel Vetter99fcb762010-02-07 16:20:18 +01001562 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1563
Eric Anholtce44b0e2008-11-06 16:00:31 -08001564 obj_priv->last_rendering_seqno = 0;
Zou Nan hai852835f2010-05-21 09:08:56 +08001565 obj_priv->ring = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07001566 if (obj_priv->active) {
1567 obj_priv->active = 0;
1568 drm_gem_object_unreference(obj);
1569 }
1570 i915_verify_inactive(dev, __FILE__, __LINE__);
1571}
1572
Daniel Vetter63560392010-02-19 11:51:59 +01001573static void
1574i915_gem_process_flushing_list(struct drm_device *dev,
Zou Nan hai852835f2010-05-21 09:08:56 +08001575 uint32_t flush_domains, uint32_t seqno,
1576 struct intel_ring_buffer *ring)
Daniel Vetter63560392010-02-19 11:51:59 +01001577{
1578 drm_i915_private_t *dev_priv = dev->dev_private;
1579 struct drm_i915_gem_object *obj_priv, *next;
1580
1581 list_for_each_entry_safe(obj_priv, next,
1582 &dev_priv->mm.gpu_write_list,
1583 gpu_write_list) {
Daniel Vettera8089e82010-04-09 19:05:09 +00001584 struct drm_gem_object *obj = &obj_priv->base;
Daniel Vetter63560392010-02-19 11:51:59 +01001585
1586 if ((obj->write_domain & flush_domains) ==
Zou Nan hai852835f2010-05-21 09:08:56 +08001587 obj->write_domain &&
1588 obj_priv->ring->ring_flag == ring->ring_flag) {
Daniel Vetter63560392010-02-19 11:51:59 +01001589 uint32_t old_write_domain = obj->write_domain;
1590
1591 obj->write_domain = 0;
1592 list_del_init(&obj_priv->gpu_write_list);
Zou Nan hai852835f2010-05-21 09:08:56 +08001593 i915_gem_object_move_to_active(obj, seqno, ring);
Daniel Vetter63560392010-02-19 11:51:59 +01001594
1595 /* update the fence lru list */
Daniel Vetter007cc8a2010-04-28 11:02:31 +02001596 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1597 struct drm_i915_fence_reg *reg =
1598 &dev_priv->fence_regs[obj_priv->fence_reg];
1599 list_move_tail(&reg->lru_list,
Daniel Vetter63560392010-02-19 11:51:59 +01001600 &dev_priv->mm.fence_list);
Daniel Vetter007cc8a2010-04-28 11:02:31 +02001601 }
Daniel Vetter63560392010-02-19 11:51:59 +01001602
1603 trace_i915_gem_object_change_domain(obj,
1604 obj->read_domains,
1605 old_write_domain);
1606 }
1607 }
1608}
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001609
Daniel Vetter5a5a0c62009-09-15 22:57:36 +02001610uint32_t
Eric Anholtb9624422009-06-03 07:27:35 +00001611i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
Zou Nan hai852835f2010-05-21 09:08:56 +08001612 uint32_t flush_domains, struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001613{
1614 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholtb9624422009-06-03 07:27:35 +00001615 struct drm_i915_file_private *i915_file_priv = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07001616 struct drm_i915_gem_request *request;
1617 uint32_t seqno;
1618 int was_empty;
Eric Anholt673a3942008-07-30 12:06:12 -07001619
Eric Anholtb9624422009-06-03 07:27:35 +00001620 if (file_priv != NULL)
1621 i915_file_priv = file_priv->driver_priv;
1622
Eric Anholt9a298b22009-03-24 12:23:04 -07001623 request = kzalloc(sizeof(*request), GFP_KERNEL);
Eric Anholt673a3942008-07-30 12:06:12 -07001624 if (request == NULL)
1625 return 0;
1626
Zou Nan hai852835f2010-05-21 09:08:56 +08001627 seqno = ring->add_request(dev, ring, file_priv, flush_domains);
Eric Anholt673a3942008-07-30 12:06:12 -07001628
1629 request->seqno = seqno;
Zou Nan hai852835f2010-05-21 09:08:56 +08001630 request->ring = ring;
Eric Anholt673a3942008-07-30 12:06:12 -07001631 request->emitted_jiffies = jiffies;
Zou Nan hai852835f2010-05-21 09:08:56 +08001632 was_empty = list_empty(&ring->request_list);
1633 list_add_tail(&request->list, &ring->request_list);
1634
Eric Anholtb9624422009-06-03 07:27:35 +00001635 if (i915_file_priv) {
1636 list_add_tail(&request->client_list,
1637 &i915_file_priv->mm.request_list);
1638 } else {
1639 INIT_LIST_HEAD(&request->client_list);
1640 }
Eric Anholt673a3942008-07-30 12:06:12 -07001641
Eric Anholtce44b0e2008-11-06 16:00:31 -08001642 /* Associate any objects on the flushing list matching the write
Daniel Vetter8bff9172010-02-11 22:19:40 +01001643 * domain we're flushing with our request.
Eric Anholtce44b0e2008-11-06 16:00:31 -08001644 */
Daniel Vetter63560392010-02-19 11:51:59 +01001645 if (flush_domains != 0)
Zou Nan hai852835f2010-05-21 09:08:56 +08001646 i915_gem_process_flushing_list(dev, flush_domains, seqno, ring);
Eric Anholtce44b0e2008-11-06 16:00:31 -08001647
Ben Gamarif65d9422009-09-14 17:48:44 -04001648 if (!dev_priv->mm.suspended) {
1649 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1650 if (was_empty)
1651 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1652 }
Eric Anholt673a3942008-07-30 12:06:12 -07001653 return seqno;
1654}
1655
1656/**
1657 * Command execution barrier
1658 *
1659 * Ensures that all commands in the ring are finished
1660 * before signalling the CPU
1661 */
Eric Anholt3043c602008-10-02 12:24:47 -07001662static uint32_t
Zou Nan hai852835f2010-05-21 09:08:56 +08001663i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001664{
Eric Anholt673a3942008-07-30 12:06:12 -07001665 uint32_t flush_domains = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07001666
1667 /* The sampler always gets flushed on i965 (sigh) */
1668 if (IS_I965G(dev))
1669 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
Zou Nan hai852835f2010-05-21 09:08:56 +08001670
1671 ring->flush(dev, ring,
1672 I915_GEM_DOMAIN_COMMAND, flush_domains);
Eric Anholt673a3942008-07-30 12:06:12 -07001673 return flush_domains;
1674}
1675
1676/**
1677 * Moves buffers associated only with the given active seqno from the active
1678 * to inactive list, potentially freeing them.
1679 */
1680static void
1681i915_gem_retire_request(struct drm_device *dev,
1682 struct drm_i915_gem_request *request)
1683{
1684 drm_i915_private_t *dev_priv = dev->dev_private;
1685
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001686 trace_i915_gem_request_retire(dev, request->seqno);
1687
Eric Anholt673a3942008-07-30 12:06:12 -07001688 /* Move any buffers on the active list that are no longer referenced
1689 * by the ringbuffer to the flushing/inactive lists as appropriate.
1690 */
Carl Worth5e118f42009-03-20 11:54:25 -07001691 spin_lock(&dev_priv->mm.active_list_lock);
Zou Nan hai852835f2010-05-21 09:08:56 +08001692 while (!list_empty(&request->ring->active_list)) {
Eric Anholt673a3942008-07-30 12:06:12 -07001693 struct drm_gem_object *obj;
1694 struct drm_i915_gem_object *obj_priv;
1695
Zou Nan hai852835f2010-05-21 09:08:56 +08001696 obj_priv = list_first_entry(&request->ring->active_list,
Eric Anholt673a3942008-07-30 12:06:12 -07001697 struct drm_i915_gem_object,
1698 list);
Daniel Vettera8089e82010-04-09 19:05:09 +00001699 obj = &obj_priv->base;
Eric Anholt673a3942008-07-30 12:06:12 -07001700
1701 /* If the seqno being retired doesn't match the oldest in the
1702 * list, then the oldest in the list must still be newer than
1703 * this seqno.
1704 */
1705 if (obj_priv->last_rendering_seqno != request->seqno)
Carl Worth5e118f42009-03-20 11:54:25 -07001706 goto out;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001707
Eric Anholt673a3942008-07-30 12:06:12 -07001708#if WATCH_LRU
1709 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1710 __func__, request->seqno, obj);
1711#endif
1712
Eric Anholtce44b0e2008-11-06 16:00:31 -08001713 if (obj->write_domain != 0)
1714 i915_gem_object_move_to_flushing(obj);
Shaohua Li68c84342009-04-08 10:58:23 +08001715 else {
1716 /* Take a reference on the object so it won't be
1717 * freed while the spinlock is held. The list
1718 * protection for this spinlock is safe when breaking
1719 * the lock like this since the next thing we do
1720 * is just get the head of the list again.
1721 */
1722 drm_gem_object_reference(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001723 i915_gem_object_move_to_inactive(obj);
Shaohua Li68c84342009-04-08 10:58:23 +08001724 spin_unlock(&dev_priv->mm.active_list_lock);
1725 drm_gem_object_unreference(obj);
1726 spin_lock(&dev_priv->mm.active_list_lock);
1727 }
Eric Anholt673a3942008-07-30 12:06:12 -07001728 }
Carl Worth5e118f42009-03-20 11:54:25 -07001729out:
1730 spin_unlock(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07001731}
1732
1733/**
1734 * Returns true if seq1 is later than seq2.
1735 */
Ben Gamari22be1722009-09-14 17:48:43 -04001736bool
Eric Anholt673a3942008-07-30 12:06:12 -07001737i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1738{
1739 return (int32_t)(seq1 - seq2) >= 0;
1740}
1741
1742uint32_t
Zou Nan hai852835f2010-05-21 09:08:56 +08001743i915_get_gem_seqno(struct drm_device *dev,
Zou Nan haid1b851f2010-05-21 09:08:57 +08001744 struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001745{
Zou Nan hai852835f2010-05-21 09:08:56 +08001746 return ring->get_gem_seqno(dev, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07001747}
1748
1749/**
1750 * This function clears the request list as sequence numbers are passed.
1751 */
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01001752static void
1753i915_gem_retire_requests_ring(struct drm_device *dev,
1754 struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001755{
1756 drm_i915_private_t *dev_priv = dev->dev_private;
1757 uint32_t seqno;
1758
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001759 if (!ring->status_page.page_addr
Zou Nan hai852835f2010-05-21 09:08:56 +08001760 || list_empty(&ring->request_list))
Karsten Wiese6c0594a2009-02-23 15:07:57 +01001761 return;
1762
Zou Nan hai852835f2010-05-21 09:08:56 +08001763 seqno = i915_get_gem_seqno(dev, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07001764
Zou Nan hai852835f2010-05-21 09:08:56 +08001765 while (!list_empty(&ring->request_list)) {
Eric Anholt673a3942008-07-30 12:06:12 -07001766 struct drm_i915_gem_request *request;
1767 uint32_t retiring_seqno;
1768
Zou Nan hai852835f2010-05-21 09:08:56 +08001769 request = list_first_entry(&ring->request_list,
Eric Anholt673a3942008-07-30 12:06:12 -07001770 struct drm_i915_gem_request,
1771 list);
1772 retiring_seqno = request->seqno;
1773
1774 if (i915_seqno_passed(seqno, retiring_seqno) ||
Ben Gamariba1234d2009-09-14 17:48:47 -04001775 atomic_read(&dev_priv->mm.wedged)) {
Eric Anholt673a3942008-07-30 12:06:12 -07001776 i915_gem_retire_request(dev, request);
1777
1778 list_del(&request->list);
Eric Anholtb9624422009-06-03 07:27:35 +00001779 list_del(&request->client_list);
Eric Anholt9a298b22009-03-24 12:23:04 -07001780 kfree(request);
Eric Anholt673a3942008-07-30 12:06:12 -07001781 } else
1782 break;
1783 }
Chris Wilson9d34e5d2009-09-24 05:26:06 +01001784
1785 if (unlikely (dev_priv->trace_irq_seqno &&
1786 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001787
1788 ring->user_irq_put(dev, ring);
Chris Wilson9d34e5d2009-09-24 05:26:06 +01001789 dev_priv->trace_irq_seqno = 0;
1790 }
Eric Anholt673a3942008-07-30 12:06:12 -07001791}
1792
1793void
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01001794i915_gem_retire_requests(struct drm_device *dev)
1795{
1796 drm_i915_private_t *dev_priv = dev->dev_private;
1797
Chris Wilsonbe726152010-07-23 23:18:50 +01001798 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1799 struct drm_i915_gem_object *obj_priv, *tmp;
1800
1801 /* We must be careful that during unbind() we do not
1802 * accidentally infinitely recurse into retire requests.
1803 * Currently:
1804 * retire -> free -> unbind -> wait -> retire_ring
1805 */
1806 list_for_each_entry_safe(obj_priv, tmp,
1807 &dev_priv->mm.deferred_free_list,
1808 list)
1809 i915_gem_free_object_tail(&obj_priv->base);
1810 }
1811
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01001812 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1813 if (HAS_BSD(dev))
1814 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1815}
1816
Daniel Vetter75ef9da2010-08-21 00:25:16 +02001817static void
Eric Anholt673a3942008-07-30 12:06:12 -07001818i915_gem_retire_work_handler(struct work_struct *work)
1819{
1820 drm_i915_private_t *dev_priv;
1821 struct drm_device *dev;
1822
1823 dev_priv = container_of(work, drm_i915_private_t,
1824 mm.retire_work.work);
1825 dev = dev_priv->dev;
1826
1827 mutex_lock(&dev->struct_mutex);
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01001828 i915_gem_retire_requests(dev);
Zou Nan haid1b851f2010-05-21 09:08:57 +08001829
Keith Packard6dbe2772008-10-14 21:41:13 -07001830 if (!dev_priv->mm.suspended &&
Zou Nan haid1b851f2010-05-21 09:08:57 +08001831 (!list_empty(&dev_priv->render_ring.request_list) ||
1832 (HAS_BSD(dev) &&
1833 !list_empty(&dev_priv->bsd_ring.request_list))))
Eric Anholt9c9fe1f2009-08-03 16:09:16 -07001834 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
Eric Anholt673a3942008-07-30 12:06:12 -07001835 mutex_unlock(&dev->struct_mutex);
1836}
1837
Daniel Vetter5a5a0c62009-09-15 22:57:36 +02001838int
Zou Nan hai852835f2010-05-21 09:08:56 +08001839i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1840 int interruptible, struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001841{
1842 drm_i915_private_t *dev_priv = dev->dev_private;
Jesse Barnes802c7eb2009-05-05 16:03:48 -07001843 u32 ier;
Eric Anholt673a3942008-07-30 12:06:12 -07001844 int ret = 0;
1845
1846 BUG_ON(seqno == 0);
1847
Daniel Vettere35a41d2010-02-11 22:13:59 +01001848 if (seqno == dev_priv->next_seqno) {
1849 seqno = i915_add_request(dev, NULL, 0, ring);
1850 if (seqno == 0)
1851 return -ENOMEM;
1852 }
1853
Ben Gamariba1234d2009-09-14 17:48:47 -04001854 if (atomic_read(&dev_priv->mm.wedged))
Ben Gamariffed1d02009-09-14 17:48:41 -04001855 return -EIO;
1856
Zou Nan hai852835f2010-05-21 09:08:56 +08001857 if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
Eric Anholtbad720f2009-10-22 16:11:14 -07001858 if (HAS_PCH_SPLIT(dev))
Zhenyu Wang036a4a72009-06-08 14:40:19 +08001859 ier = I915_READ(DEIER) | I915_READ(GTIER);
1860 else
1861 ier = I915_READ(IER);
Jesse Barnes802c7eb2009-05-05 16:03:48 -07001862 if (!ier) {
1863 DRM_ERROR("something (likely vbetool) disabled "
1864 "interrupts, re-enabling\n");
1865 i915_driver_irq_preinstall(dev);
1866 i915_driver_irq_postinstall(dev);
1867 }
1868
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001869 trace_i915_gem_request_wait_begin(dev, seqno);
1870
Zou Nan hai852835f2010-05-21 09:08:56 +08001871 ring->waiting_gem_seqno = seqno;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001872 ring->user_irq_get(dev, ring);
Daniel Vetter48764bf2009-09-15 22:57:32 +02001873 if (interruptible)
Zou Nan hai852835f2010-05-21 09:08:56 +08001874 ret = wait_event_interruptible(ring->irq_queue,
1875 i915_seqno_passed(
1876 ring->get_gem_seqno(dev, ring), seqno)
1877 || atomic_read(&dev_priv->mm.wedged));
Daniel Vetter48764bf2009-09-15 22:57:32 +02001878 else
Zou Nan hai852835f2010-05-21 09:08:56 +08001879 wait_event(ring->irq_queue,
1880 i915_seqno_passed(
1881 ring->get_gem_seqno(dev, ring), seqno)
1882 || atomic_read(&dev_priv->mm.wedged));
Daniel Vetter48764bf2009-09-15 22:57:32 +02001883
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001884 ring->user_irq_put(dev, ring);
Zou Nan hai852835f2010-05-21 09:08:56 +08001885 ring->waiting_gem_seqno = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001886
1887 trace_i915_gem_request_wait_end(dev, seqno);
Eric Anholt673a3942008-07-30 12:06:12 -07001888 }
Ben Gamariba1234d2009-09-14 17:48:47 -04001889 if (atomic_read(&dev_priv->mm.wedged))
Eric Anholt673a3942008-07-30 12:06:12 -07001890 ret = -EIO;
1891
1892 if (ret && ret != -ERESTARTSYS)
Daniel Vetter8bff9172010-02-11 22:19:40 +01001893 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
1894 __func__, ret, seqno, ring->get_gem_seqno(dev, ring),
1895 dev_priv->next_seqno);
Eric Anholt673a3942008-07-30 12:06:12 -07001896
1897 /* Directly dispatch request retiring. While we have the work queue
1898 * to handle this, the waiter on a request often wants an associated
1899 * buffer to have made it to the inactive list, and we would need
1900 * a separate wait queue to handle that.
1901 */
1902 if (ret == 0)
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01001903 i915_gem_retire_requests_ring(dev, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07001904
1905 return ret;
1906}
1907
Daniel Vetter48764bf2009-09-15 22:57:32 +02001908/**
1909 * Waits for a sequence number to be signaled, and cleans up the
1910 * request and object lists appropriately for that event.
1911 */
1912static int
Zou Nan hai852835f2010-05-21 09:08:56 +08001913i915_wait_request(struct drm_device *dev, uint32_t seqno,
1914 struct intel_ring_buffer *ring)
Daniel Vetter48764bf2009-09-15 22:57:32 +02001915{
Zou Nan hai852835f2010-05-21 09:08:56 +08001916 return i915_do_wait_request(dev, seqno, 1, ring);
Daniel Vetter48764bf2009-09-15 22:57:32 +02001917}
1918
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001919static void
1920i915_gem_flush(struct drm_device *dev,
1921 uint32_t invalidate_domains,
1922 uint32_t flush_domains)
1923{
1924 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter8bff9172010-02-11 22:19:40 +01001925
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001926 if (flush_domains & I915_GEM_DOMAIN_CPU)
1927 drm_agp_chipset_flush(dev);
Daniel Vetter8bff9172010-02-11 22:19:40 +01001928
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001929 dev_priv->render_ring.flush(dev, &dev_priv->render_ring,
1930 invalidate_domains,
1931 flush_domains);
Zou Nan haid1b851f2010-05-21 09:08:57 +08001932
1933 if (HAS_BSD(dev))
1934 dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring,
1935 invalidate_domains,
1936 flush_domains);
Daniel Vetter8bff9172010-02-11 22:19:40 +01001937
1938 /* Associate any objects on the flushing list matching the write
1939 * domain we're flushing with the next request.
1940 */
1941 if (flush_domains != 0) {
1942 i915_gem_process_flushing_list(dev, flush_domains, 0,
1943 &dev_priv->render_ring);
1944 if (HAS_BSD(dev))
1945 i915_gem_process_flushing_list(dev, flush_domains, 0,
1946 &dev_priv->bsd_ring);
1947 }
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001948}
1949
Eric Anholt673a3942008-07-30 12:06:12 -07001950/**
1951 * Ensures that all rendering to the object has completed and the object is
1952 * safe to unbind from the GTT or access from the CPU.
1953 */
1954static int
Daniel Vettere35a41d2010-02-11 22:13:59 +01001955i915_gem_object_wait_rendering(struct drm_gem_object *obj,
1956 bool interruptible)
Eric Anholt673a3942008-07-30 12:06:12 -07001957{
1958 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001959 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001960 int ret;
1961
Eric Anholte47c68e2008-11-14 13:35:19 -08001962 /* This function only exists to support waiting for existing rendering,
1963 * not for emitting required flushes.
Eric Anholt673a3942008-07-30 12:06:12 -07001964 */
Eric Anholte47c68e2008-11-14 13:35:19 -08001965 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
Eric Anholt673a3942008-07-30 12:06:12 -07001966
1967 /* If there is rendering queued on the buffer being evicted, wait for
1968 * it.
1969 */
1970 if (obj_priv->active) {
1971#if WATCH_BUF
1972 DRM_INFO("%s: object %p wait for seqno %08x\n",
1973 __func__, obj, obj_priv->last_rendering_seqno);
1974#endif
Daniel Vettere35a41d2010-02-11 22:13:59 +01001975 ret = i915_do_wait_request(dev,
1976 obj_priv->last_rendering_seqno,
1977 interruptible,
1978 obj_priv->ring);
Eric Anholt673a3942008-07-30 12:06:12 -07001979 if (ret != 0)
1980 return ret;
1981 }
1982
1983 return 0;
1984}
1985
1986/**
1987 * Unbinds an object from the GTT aperture.
1988 */
Jesse Barnes0f973f22009-01-26 17:10:45 -08001989int
Eric Anholt673a3942008-07-30 12:06:12 -07001990i915_gem_object_unbind(struct drm_gem_object *obj)
1991{
1992 struct drm_device *dev = obj->dev;
Daniel Vetter4a87b8c2010-02-19 11:51:57 +01001993 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001994 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001995 int ret = 0;
1996
1997#if WATCH_BUF
1998 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1999 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
2000#endif
2001 if (obj_priv->gtt_space == NULL)
2002 return 0;
2003
2004 if (obj_priv->pin_count != 0) {
2005 DRM_ERROR("Attempting to unbind pinned buffer\n");
2006 return -EINVAL;
2007 }
2008
Eric Anholt5323fd02009-09-09 11:50:45 -07002009 /* blow away mappings if mapped through GTT */
2010 i915_gem_release_mmap(obj);
2011
Eric Anholt673a3942008-07-30 12:06:12 -07002012 /* Move the object to the CPU domain to ensure that
2013 * any possible CPU writes while it's not in the GTT
2014 * are flushed when we go to remap it. This will
2015 * also ensure that all pending GPU writes are finished
2016 * before we unbind.
2017 */
Eric Anholte47c68e2008-11-14 13:35:19 -08002018 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
Chris Wilson8dc17752010-07-23 23:18:51 +01002019 if (ret == -ERESTARTSYS)
Eric Anholt673a3942008-07-30 12:06:12 -07002020 return ret;
Chris Wilson8dc17752010-07-23 23:18:51 +01002021 /* Continue on if we fail due to EIO, the GPU is hung so we
2022 * should be safe and we need to cleanup or else we might
2023 * cause memory corruption through use-after-free.
2024 */
Eric Anholt673a3942008-07-30 12:06:12 -07002025
Daniel Vetter96b47b62009-12-15 17:50:00 +01002026 /* release the fence reg _after_ flushing */
2027 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2028 i915_gem_clear_fence_reg(obj);
2029
Eric Anholt673a3942008-07-30 12:06:12 -07002030 if (obj_priv->agp_mem != NULL) {
2031 drm_unbind_agp(obj_priv->agp_mem);
2032 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2033 obj_priv->agp_mem = NULL;
2034 }
2035
Eric Anholt856fa192009-03-19 14:10:50 -07002036 i915_gem_object_put_pages(obj);
Chris Wilsona32808c2009-09-20 21:29:47 +01002037 BUG_ON(obj_priv->pages_refcount);
Eric Anholt673a3942008-07-30 12:06:12 -07002038
2039 if (obj_priv->gtt_space) {
2040 atomic_dec(&dev->gtt_count);
2041 atomic_sub(obj->size, &dev->gtt_memory);
2042
2043 drm_mm_put_block(obj_priv->gtt_space);
2044 obj_priv->gtt_space = NULL;
2045 }
2046
2047 /* Remove ourselves from the LRU list if present. */
Daniel Vetter4a87b8c2010-02-19 11:51:57 +01002048 spin_lock(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07002049 if (!list_empty(&obj_priv->list))
2050 list_del_init(&obj_priv->list);
Daniel Vetter4a87b8c2010-02-19 11:51:57 +01002051 spin_unlock(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07002052
Chris Wilson963b4832009-09-20 23:03:54 +01002053 if (i915_gem_object_is_purgeable(obj_priv))
2054 i915_gem_object_truncate(obj);
2055
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002056 trace_i915_gem_object_unbind(obj);
2057
Chris Wilson8dc17752010-07-23 23:18:51 +01002058 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002059}
2060
Chris Wilsonb47eb4a2010-08-07 11:01:23 +01002061int
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002062i915_gpu_idle(struct drm_device *dev)
2063{
2064 drm_i915_private_t *dev_priv = dev->dev_private;
2065 bool lists_empty;
Zou Nan haid1b851f2010-05-21 09:08:57 +08002066 uint32_t seqno1, seqno2;
Zou Nan hai852835f2010-05-21 09:08:56 +08002067 int ret;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002068
2069 spin_lock(&dev_priv->mm.active_list_lock);
Zou Nan haid1b851f2010-05-21 09:08:57 +08002070 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2071 list_empty(&dev_priv->render_ring.active_list) &&
2072 (!HAS_BSD(dev) ||
2073 list_empty(&dev_priv->bsd_ring.active_list)));
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002074 spin_unlock(&dev_priv->mm.active_list_lock);
2075
2076 if (lists_empty)
2077 return 0;
2078
2079 /* Flush everything onto the inactive list. */
2080 i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
Daniel Vetter8bff9172010-02-11 22:19:40 +01002081 seqno1 = i915_add_request(dev, NULL, 0,
Zou Nan hai852835f2010-05-21 09:08:56 +08002082 &dev_priv->render_ring);
Zou Nan haid1b851f2010-05-21 09:08:57 +08002083 if (seqno1 == 0)
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002084 return -ENOMEM;
Zou Nan haid1b851f2010-05-21 09:08:57 +08002085 ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring);
2086
2087 if (HAS_BSD(dev)) {
Daniel Vetter8bff9172010-02-11 22:19:40 +01002088 seqno2 = i915_add_request(dev, NULL, 0,
Zou Nan haid1b851f2010-05-21 09:08:57 +08002089 &dev_priv->bsd_ring);
2090 if (seqno2 == 0)
2091 return -ENOMEM;
2092
2093 ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring);
2094 if (ret)
2095 return ret;
2096 }
2097
Zou Nan hai852835f2010-05-21 09:08:56 +08002098 return ret;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002099}
2100
Ben Gamari6911a9b2009-04-02 11:24:54 -07002101int
Chris Wilson4bdadb92010-01-27 13:36:32 +00002102i915_gem_object_get_pages(struct drm_gem_object *obj,
2103 gfp_t gfpmask)
Eric Anholt673a3942008-07-30 12:06:12 -07002104{
Daniel Vetter23010e42010-03-08 13:35:02 +01002105 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002106 int page_count, i;
2107 struct address_space *mapping;
2108 struct inode *inode;
2109 struct page *page;
Eric Anholt673a3942008-07-30 12:06:12 -07002110
Daniel Vetter778c3542010-05-13 11:49:44 +02002111 BUG_ON(obj_priv->pages_refcount
2112 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2113
Eric Anholt856fa192009-03-19 14:10:50 -07002114 if (obj_priv->pages_refcount++ != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07002115 return 0;
2116
2117 /* Get the list of pages out of our struct file. They'll be pinned
2118 * at this point until we release them.
2119 */
2120 page_count = obj->size / PAGE_SIZE;
Eric Anholt856fa192009-03-19 14:10:50 -07002121 BUG_ON(obj_priv->pages != NULL);
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07002122 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
Eric Anholt856fa192009-03-19 14:10:50 -07002123 if (obj_priv->pages == NULL) {
Eric Anholt856fa192009-03-19 14:10:50 -07002124 obj_priv->pages_refcount--;
Eric Anholt673a3942008-07-30 12:06:12 -07002125 return -ENOMEM;
2126 }
2127
2128 inode = obj->filp->f_path.dentry->d_inode;
2129 mapping = inode->i_mapping;
2130 for (i = 0; i < page_count; i++) {
Chris Wilson4bdadb92010-01-27 13:36:32 +00002131 page = read_cache_page_gfp(mapping, i,
Linus Torvalds985b8232010-07-02 10:04:42 +10002132 GFP_HIGHUSER |
Chris Wilson4bdadb92010-01-27 13:36:32 +00002133 __GFP_COLD |
Linus Torvaldscd9f0402010-07-18 09:44:37 -07002134 __GFP_RECLAIMABLE |
Chris Wilson4bdadb92010-01-27 13:36:32 +00002135 gfpmask);
Chris Wilson1f2b1012010-03-12 19:52:55 +00002136 if (IS_ERR(page))
2137 goto err_pages;
2138
Eric Anholt856fa192009-03-19 14:10:50 -07002139 obj_priv->pages[i] = page;
Eric Anholt673a3942008-07-30 12:06:12 -07002140 }
Eric Anholt280b7132009-03-12 16:56:27 -07002141
2142 if (obj_priv->tiling_mode != I915_TILING_NONE)
2143 i915_gem_object_do_bit_17_swizzle(obj);
2144
Eric Anholt673a3942008-07-30 12:06:12 -07002145 return 0;
Chris Wilson1f2b1012010-03-12 19:52:55 +00002146
2147err_pages:
2148 while (i--)
2149 page_cache_release(obj_priv->pages[i]);
2150
2151 drm_free_large(obj_priv->pages);
2152 obj_priv->pages = NULL;
2153 obj_priv->pages_refcount--;
2154 return PTR_ERR(page);
Eric Anholt673a3942008-07-30 12:06:12 -07002155}
2156
Eric Anholt4e901fd2009-10-26 16:44:17 -07002157static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2158{
2159 struct drm_gem_object *obj = reg->obj;
2160 struct drm_device *dev = obj->dev;
2161 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002162 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt4e901fd2009-10-26 16:44:17 -07002163 int regnum = obj_priv->fence_reg;
2164 uint64_t val;
2165
2166 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2167 0xfffff000) << 32;
2168 val |= obj_priv->gtt_offset & 0xfffff000;
2169 val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2170 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2171
2172 if (obj_priv->tiling_mode == I915_TILING_Y)
2173 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2174 val |= I965_FENCE_REG_VALID;
2175
2176 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2177}
2178
Jesse Barnesde151cf2008-11-12 10:03:55 -08002179static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2180{
2181 struct drm_gem_object *obj = reg->obj;
2182 struct drm_device *dev = obj->dev;
2183 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002184 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002185 int regnum = obj_priv->fence_reg;
2186 uint64_t val;
2187
2188 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2189 0xfffff000) << 32;
2190 val |= obj_priv->gtt_offset & 0xfffff000;
2191 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2192 if (obj_priv->tiling_mode == I915_TILING_Y)
2193 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2194 val |= I965_FENCE_REG_VALID;
2195
2196 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2197}
2198
2199static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2200{
2201 struct drm_gem_object *obj = reg->obj;
2202 struct drm_device *dev = obj->dev;
2203 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002204 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002205 int regnum = obj_priv->fence_reg;
Jesse Barnes0f973f22009-01-26 17:10:45 -08002206 int tile_width;
Eric Anholtdc529a42009-03-10 22:34:49 -07002207 uint32_t fence_reg, val;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002208 uint32_t pitch_val;
2209
2210 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2211 (obj_priv->gtt_offset & (obj->size - 1))) {
Linus Torvaldsf06da262009-02-09 08:57:29 -08002212 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
Jesse Barnes0f973f22009-01-26 17:10:45 -08002213 __func__, obj_priv->gtt_offset, obj->size);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002214 return;
2215 }
2216
Jesse Barnes0f973f22009-01-26 17:10:45 -08002217 if (obj_priv->tiling_mode == I915_TILING_Y &&
2218 HAS_128_BYTE_Y_TILING(dev))
2219 tile_width = 128;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002220 else
Jesse Barnes0f973f22009-01-26 17:10:45 -08002221 tile_width = 512;
2222
2223 /* Note: pitch better be a power of two tile widths */
2224 pitch_val = obj_priv->stride / tile_width;
2225 pitch_val = ffs(pitch_val) - 1;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002226
Daniel Vetterc36a2a62010-04-17 15:12:03 +02002227 if (obj_priv->tiling_mode == I915_TILING_Y &&
2228 HAS_128_BYTE_Y_TILING(dev))
2229 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2230 else
2231 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2232
Jesse Barnesde151cf2008-11-12 10:03:55 -08002233 val = obj_priv->gtt_offset;
2234 if (obj_priv->tiling_mode == I915_TILING_Y)
2235 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2236 val |= I915_FENCE_SIZE_BITS(obj->size);
2237 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2238 val |= I830_FENCE_REG_VALID;
2239
Eric Anholtdc529a42009-03-10 22:34:49 -07002240 if (regnum < 8)
2241 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2242 else
2243 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2244 I915_WRITE(fence_reg, val);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002245}
2246
2247static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2248{
2249 struct drm_gem_object *obj = reg->obj;
2250 struct drm_device *dev = obj->dev;
2251 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002252 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002253 int regnum = obj_priv->fence_reg;
2254 uint32_t val;
2255 uint32_t pitch_val;
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002256 uint32_t fence_size_bits;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002257
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002258 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
Jesse Barnesde151cf2008-11-12 10:03:55 -08002259 (obj_priv->gtt_offset & (obj->size - 1))) {
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002260 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
Jesse Barnes0f973f22009-01-26 17:10:45 -08002261 __func__, obj_priv->gtt_offset);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002262 return;
2263 }
2264
Eric Anholte76a16d2009-05-26 17:44:56 -07002265 pitch_val = obj_priv->stride / 128;
2266 pitch_val = ffs(pitch_val) - 1;
2267 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2268
Jesse Barnesde151cf2008-11-12 10:03:55 -08002269 val = obj_priv->gtt_offset;
2270 if (obj_priv->tiling_mode == I915_TILING_Y)
2271 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002272 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2273 WARN_ON(fence_size_bits & ~0x00000f00);
2274 val |= fence_size_bits;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002275 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2276 val |= I830_FENCE_REG_VALID;
2277
2278 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002279}
2280
Daniel Vetterae3db242010-02-19 11:51:58 +01002281static int i915_find_fence_reg(struct drm_device *dev)
2282{
2283 struct drm_i915_fence_reg *reg = NULL;
2284 struct drm_i915_gem_object *obj_priv = NULL;
2285 struct drm_i915_private *dev_priv = dev->dev_private;
2286 struct drm_gem_object *obj = NULL;
2287 int i, avail, ret;
2288
2289 /* First try to find a free reg */
2290 avail = 0;
2291 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2292 reg = &dev_priv->fence_regs[i];
2293 if (!reg->obj)
2294 return i;
2295
Daniel Vetter23010e42010-03-08 13:35:02 +01002296 obj_priv = to_intel_bo(reg->obj);
Daniel Vetterae3db242010-02-19 11:51:58 +01002297 if (!obj_priv->pin_count)
2298 avail++;
2299 }
2300
2301 if (avail == 0)
2302 return -ENOSPC;
2303
2304 /* None available, try to steal one or wait for a user to finish */
2305 i = I915_FENCE_REG_NONE;
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002306 list_for_each_entry(reg, &dev_priv->mm.fence_list,
2307 lru_list) {
2308 obj = reg->obj;
2309 obj_priv = to_intel_bo(obj);
Daniel Vetterae3db242010-02-19 11:51:58 +01002310
2311 if (obj_priv->pin_count)
2312 continue;
2313
2314 /* found one! */
2315 i = obj_priv->fence_reg;
2316 break;
2317 }
2318
2319 BUG_ON(i == I915_FENCE_REG_NONE);
2320
2321 /* We only have a reference on obj from the active list. put_fence_reg
2322 * might drop that one, causing a use-after-free in it. So hold a
2323 * private reference to obj like the other callers of put_fence_reg
2324 * (set_tiling ioctl) do. */
2325 drm_gem_object_reference(obj);
2326 ret = i915_gem_object_put_fence_reg(obj);
2327 drm_gem_object_unreference(obj);
2328 if (ret != 0)
2329 return ret;
2330
2331 return i;
2332}
2333
Jesse Barnesde151cf2008-11-12 10:03:55 -08002334/**
2335 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2336 * @obj: object to map through a fence reg
2337 *
2338 * When mapping objects through the GTT, userspace wants to be able to write
2339 * to them without having to worry about swizzling if the object is tiled.
2340 *
2341 * This function walks the fence regs looking for a free one for @obj,
2342 * stealing one if it can't find any.
2343 *
2344 * It then sets up the reg based on the object's properties: address, pitch
2345 * and tiling format.
2346 */
Chris Wilson8c4b8c32009-06-17 22:08:52 +01002347int
2348i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002349{
2350 struct drm_device *dev = obj->dev;
Jesse Barnes79e53942008-11-07 14:24:08 -08002351 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002352 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002353 struct drm_i915_fence_reg *reg = NULL;
Daniel Vetterae3db242010-02-19 11:51:58 +01002354 int ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002355
Eric Anholta09ba7f2009-08-29 12:49:51 -07002356 /* Just update our place in the LRU if our fence is getting used. */
2357 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002358 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2359 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07002360 return 0;
2361 }
2362
Jesse Barnesde151cf2008-11-12 10:03:55 -08002363 switch (obj_priv->tiling_mode) {
2364 case I915_TILING_NONE:
2365 WARN(1, "allocating a fence for non-tiled object?\n");
2366 break;
2367 case I915_TILING_X:
Jesse Barnes0f973f22009-01-26 17:10:45 -08002368 if (!obj_priv->stride)
2369 return -EINVAL;
2370 WARN((obj_priv->stride & (512 - 1)),
2371 "object 0x%08x is X tiled but has non-512B pitch\n",
2372 obj_priv->gtt_offset);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002373 break;
2374 case I915_TILING_Y:
Jesse Barnes0f973f22009-01-26 17:10:45 -08002375 if (!obj_priv->stride)
2376 return -EINVAL;
2377 WARN((obj_priv->stride & (128 - 1)),
2378 "object 0x%08x is Y tiled but has non-128B pitch\n",
2379 obj_priv->gtt_offset);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002380 break;
2381 }
2382
Daniel Vetterae3db242010-02-19 11:51:58 +01002383 ret = i915_find_fence_reg(dev);
2384 if (ret < 0)
2385 return ret;
Chris Wilsonfc7170b2009-02-11 14:26:46 +00002386
Daniel Vetterae3db242010-02-19 11:51:58 +01002387 obj_priv->fence_reg = ret;
2388 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002389 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07002390
Jesse Barnesde151cf2008-11-12 10:03:55 -08002391 reg->obj = obj;
2392
Eric Anholt4e901fd2009-10-26 16:44:17 -07002393 if (IS_GEN6(dev))
2394 sandybridge_write_fence_reg(reg);
2395 else if (IS_I965G(dev))
Jesse Barnesde151cf2008-11-12 10:03:55 -08002396 i965_write_fence_reg(reg);
2397 else if (IS_I9XX(dev))
2398 i915_write_fence_reg(reg);
2399 else
2400 i830_write_fence_reg(reg);
Eric Anholtd9ddcb92009-01-27 10:33:49 -08002401
Daniel Vetterae3db242010-02-19 11:51:58 +01002402 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2403 obj_priv->tiling_mode);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002404
Eric Anholtd9ddcb92009-01-27 10:33:49 -08002405 return 0;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002406}
2407
2408/**
2409 * i915_gem_clear_fence_reg - clear out fence register info
2410 * @obj: object to clear
2411 *
2412 * Zeroes out the fence register itself and clears out the associated
2413 * data structures in dev_priv and obj_priv.
2414 */
2415static void
2416i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2417{
2418 struct drm_device *dev = obj->dev;
Jesse Barnes79e53942008-11-07 14:24:08 -08002419 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002420 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002421 struct drm_i915_fence_reg *reg =
2422 &dev_priv->fence_regs[obj_priv->fence_reg];
Jesse Barnesde151cf2008-11-12 10:03:55 -08002423
Eric Anholt4e901fd2009-10-26 16:44:17 -07002424 if (IS_GEN6(dev)) {
2425 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2426 (obj_priv->fence_reg * 8), 0);
2427 } else if (IS_I965G(dev)) {
Jesse Barnesde151cf2008-11-12 10:03:55 -08002428 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
Eric Anholt4e901fd2009-10-26 16:44:17 -07002429 } else {
Eric Anholtdc529a42009-03-10 22:34:49 -07002430 uint32_t fence_reg;
2431
2432 if (obj_priv->fence_reg < 8)
2433 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2434 else
2435 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2436 8) * 4;
2437
2438 I915_WRITE(fence_reg, 0);
2439 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08002440
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002441 reg->obj = NULL;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002442 obj_priv->fence_reg = I915_FENCE_REG_NONE;
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002443 list_del_init(&reg->lru_list);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002444}
2445
Eric Anholt673a3942008-07-30 12:06:12 -07002446/**
Chris Wilson52dc7d32009-06-06 09:46:01 +01002447 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2448 * to the buffer to finish, and then resets the fence register.
2449 * @obj: tiled object holding a fence register.
2450 *
2451 * Zeroes out the fence register itself and clears out the associated
2452 * data structures in dev_priv and obj_priv.
2453 */
2454int
2455i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2456{
2457 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01002458 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson52dc7d32009-06-06 09:46:01 +01002459
2460 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2461 return 0;
2462
Daniel Vetter10ae9bd2010-02-01 13:59:17 +01002463 /* If we've changed tiling, GTT-mappings of the object
2464 * need to re-fault to ensure that the correct fence register
2465 * setup is in place.
2466 */
2467 i915_gem_release_mmap(obj);
2468
Chris Wilson52dc7d32009-06-06 09:46:01 +01002469 /* On the i915, GPU access to tiled buffers is via a fence,
2470 * therefore we must wait for any outstanding access to complete
2471 * before clearing the fence.
2472 */
2473 if (!IS_I965G(dev)) {
2474 int ret;
2475
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002476 ret = i915_gem_object_flush_gpu_write_domain(obj);
2477 if (ret != 0)
2478 return ret;
2479
Daniel Vettere35a41d2010-02-11 22:13:59 +01002480 ret = i915_gem_object_wait_rendering(obj, true);
Chris Wilson52dc7d32009-06-06 09:46:01 +01002481 if (ret != 0)
2482 return ret;
2483 }
2484
Daniel Vetter4a726612010-02-01 13:59:16 +01002485 i915_gem_object_flush_gtt_write_domain(obj);
Chris Wilson52dc7d32009-06-06 09:46:01 +01002486 i915_gem_clear_fence_reg (obj);
2487
2488 return 0;
2489}
2490
2491/**
Eric Anholt673a3942008-07-30 12:06:12 -07002492 * Finds free space in the GTT aperture and binds the object there.
2493 */
2494static int
2495i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2496{
2497 struct drm_device *dev = obj->dev;
2498 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002499 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002500 struct drm_mm_node *free_space;
Chris Wilson4bdadb92010-01-27 13:36:32 +00002501 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
Chris Wilson07f73f62009-09-14 16:50:30 +01002502 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002503
Chris Wilsonbb6baf72009-09-22 14:24:13 +01002504 if (obj_priv->madv != I915_MADV_WILLNEED) {
Chris Wilson3ef94da2009-09-14 16:50:29 +01002505 DRM_ERROR("Attempting to bind a purgeable object\n");
2506 return -EINVAL;
2507 }
2508
Eric Anholt673a3942008-07-30 12:06:12 -07002509 if (alignment == 0)
Jesse Barnes0f973f22009-01-26 17:10:45 -08002510 alignment = i915_gem_get_gtt_alignment(obj);
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002511 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
Eric Anholt673a3942008-07-30 12:06:12 -07002512 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2513 return -EINVAL;
2514 }
2515
Chris Wilson654fc602010-05-27 13:18:21 +01002516 /* If the object is bigger than the entire aperture, reject it early
2517 * before evicting everything in a vain attempt to find space.
2518 */
2519 if (obj->size > dev->gtt_total) {
2520 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2521 return -E2BIG;
2522 }
2523
Eric Anholt673a3942008-07-30 12:06:12 -07002524 search_free:
2525 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2526 obj->size, alignment, 0);
2527 if (free_space != NULL) {
2528 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2529 alignment);
Daniel Vetterdb3307a2010-07-02 15:02:12 +01002530 if (obj_priv->gtt_space != NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07002531 obj_priv->gtt_offset = obj_priv->gtt_space->start;
Eric Anholt673a3942008-07-30 12:06:12 -07002532 }
2533 if (obj_priv->gtt_space == NULL) {
2534 /* If the gtt is empty and we're still having trouble
2535 * fitting our object in, we're out of memory.
2536 */
2537#if WATCH_LRU
2538 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2539#endif
Daniel Vetter0108a3e2010-08-07 11:01:21 +01002540 ret = i915_gem_evict_something(dev, obj->size, alignment);
Chris Wilson97311292009-09-21 00:22:34 +01002541 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07002542 return ret;
Chris Wilson97311292009-09-21 00:22:34 +01002543
Eric Anholt673a3942008-07-30 12:06:12 -07002544 goto search_free;
2545 }
2546
2547#if WATCH_BUF
Krzysztof Halasacfd43c02009-06-20 00:31:28 +02002548 DRM_INFO("Binding object of size %zd at 0x%08x\n",
Eric Anholt673a3942008-07-30 12:06:12 -07002549 obj->size, obj_priv->gtt_offset);
2550#endif
Chris Wilson4bdadb92010-01-27 13:36:32 +00002551 ret = i915_gem_object_get_pages(obj, gfpmask);
Eric Anholt673a3942008-07-30 12:06:12 -07002552 if (ret) {
2553 drm_mm_put_block(obj_priv->gtt_space);
2554 obj_priv->gtt_space = NULL;
Chris Wilson07f73f62009-09-14 16:50:30 +01002555
2556 if (ret == -ENOMEM) {
2557 /* first try to clear up some space from the GTT */
Daniel Vetter0108a3e2010-08-07 11:01:21 +01002558 ret = i915_gem_evict_something(dev, obj->size,
2559 alignment);
Chris Wilson07f73f62009-09-14 16:50:30 +01002560 if (ret) {
Chris Wilson07f73f62009-09-14 16:50:30 +01002561 /* now try to shrink everyone else */
Chris Wilson4bdadb92010-01-27 13:36:32 +00002562 if (gfpmask) {
2563 gfpmask = 0;
2564 goto search_free;
Chris Wilson07f73f62009-09-14 16:50:30 +01002565 }
2566
2567 return ret;
2568 }
2569
2570 goto search_free;
2571 }
2572
Eric Anholt673a3942008-07-30 12:06:12 -07002573 return ret;
2574 }
2575
Eric Anholt673a3942008-07-30 12:06:12 -07002576 /* Create an AGP memory structure pointing at our pages, and bind it
2577 * into the GTT.
2578 */
2579 obj_priv->agp_mem = drm_agp_bind_pages(dev,
Eric Anholt856fa192009-03-19 14:10:50 -07002580 obj_priv->pages,
Chris Wilson07f73f62009-09-14 16:50:30 +01002581 obj->size >> PAGE_SHIFT,
Keith Packardba1eb1d2008-10-14 19:55:10 -07002582 obj_priv->gtt_offset,
2583 obj_priv->agp_type);
Eric Anholt673a3942008-07-30 12:06:12 -07002584 if (obj_priv->agp_mem == NULL) {
Eric Anholt856fa192009-03-19 14:10:50 -07002585 i915_gem_object_put_pages(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002586 drm_mm_put_block(obj_priv->gtt_space);
2587 obj_priv->gtt_space = NULL;
Chris Wilson07f73f62009-09-14 16:50:30 +01002588
Daniel Vetter0108a3e2010-08-07 11:01:21 +01002589 ret = i915_gem_evict_something(dev, obj->size, alignment);
Chris Wilson97311292009-09-21 00:22:34 +01002590 if (ret)
Chris Wilson07f73f62009-09-14 16:50:30 +01002591 return ret;
Chris Wilson07f73f62009-09-14 16:50:30 +01002592
2593 goto search_free;
Eric Anholt673a3942008-07-30 12:06:12 -07002594 }
2595 atomic_inc(&dev->gtt_count);
2596 atomic_add(obj->size, &dev->gtt_memory);
2597
Chris Wilsonbf1a1092010-08-07 11:01:20 +01002598 /* keep track of bounds object by adding it to the inactive list */
2599 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2600
Eric Anholt673a3942008-07-30 12:06:12 -07002601 /* Assert that the object is not currently in any GPU domain. As it
2602 * wasn't in the GTT, there shouldn't be any way it could have been in
2603 * a GPU cache
2604 */
Chris Wilson21d509e2009-06-06 09:46:02 +01002605 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2606 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
Eric Anholt673a3942008-07-30 12:06:12 -07002607
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002608 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2609
Eric Anholt673a3942008-07-30 12:06:12 -07002610 return 0;
2611}
2612
2613void
2614i915_gem_clflush_object(struct drm_gem_object *obj)
2615{
Daniel Vetter23010e42010-03-08 13:35:02 +01002616 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002617
2618 /* If we don't have a page list set up, then we're not pinned
2619 * to GPU, and we can ignore the cache flush because it'll happen
2620 * again at bind time.
2621 */
Eric Anholt856fa192009-03-19 14:10:50 -07002622 if (obj_priv->pages == NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07002623 return;
2624
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002625 trace_i915_gem_object_clflush(obj);
Eric Anholtcfa16a02009-05-26 18:46:16 -07002626
Eric Anholt856fa192009-03-19 14:10:50 -07002627 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
Eric Anholt673a3942008-07-30 12:06:12 -07002628}
2629
Eric Anholte47c68e2008-11-14 13:35:19 -08002630/** Flushes any GPU write domain for the object if it's dirty. */
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002631static int
Eric Anholte47c68e2008-11-14 13:35:19 -08002632i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2633{
2634 struct drm_device *dev = obj->dev;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002635 uint32_t old_write_domain;
Zou Nan hai852835f2010-05-21 09:08:56 +08002636 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08002637
2638 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002639 return 0;
Eric Anholte47c68e2008-11-14 13:35:19 -08002640
2641 /* Queue the GPU write cache flushing we need. */
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002642 old_write_domain = obj->write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002643 i915_gem_flush(dev, 0, obj->write_domain);
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002644 if (i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring) == 0)
2645 return -ENOMEM;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002646
2647 trace_i915_gem_object_change_domain(obj,
2648 obj->read_domains,
2649 old_write_domain);
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002650 return 0;
Eric Anholte47c68e2008-11-14 13:35:19 -08002651}
2652
2653/** Flushes the GTT write domain for the object if it's dirty. */
2654static void
2655i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2656{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002657 uint32_t old_write_domain;
2658
Eric Anholte47c68e2008-11-14 13:35:19 -08002659 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2660 return;
2661
2662 /* No actual flushing is required for the GTT write domain. Writes
2663 * to it immediately go to main memory as far as we know, so there's
2664 * no chipset flush. It also doesn't land in render cache.
2665 */
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002666 old_write_domain = obj->write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002667 obj->write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002668
2669 trace_i915_gem_object_change_domain(obj,
2670 obj->read_domains,
2671 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08002672}
2673
2674/** Flushes the CPU write domain for the object if it's dirty. */
2675static void
2676i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2677{
2678 struct drm_device *dev = obj->dev;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002679 uint32_t old_write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002680
2681 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2682 return;
2683
2684 i915_gem_clflush_object(obj);
2685 drm_agp_chipset_flush(dev);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002686 old_write_domain = obj->write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002687 obj->write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002688
2689 trace_i915_gem_object_change_domain(obj,
2690 obj->read_domains,
2691 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08002692}
2693
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002694int
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05002695i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2696{
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002697 int ret = 0;
2698
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05002699 switch (obj->write_domain) {
2700 case I915_GEM_DOMAIN_GTT:
2701 i915_gem_object_flush_gtt_write_domain(obj);
2702 break;
2703 case I915_GEM_DOMAIN_CPU:
2704 i915_gem_object_flush_cpu_write_domain(obj);
2705 break;
2706 default:
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002707 ret = i915_gem_object_flush_gpu_write_domain(obj);
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05002708 break;
2709 }
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002710
2711 return ret;
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05002712}
2713
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002714/**
2715 * Moves a single object to the GTT read, and possibly write domain.
2716 *
2717 * This function returns when the move is complete, including waiting on
2718 * flushes to occur.
2719 */
Jesse Barnes79e53942008-11-07 14:24:08 -08002720int
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002721i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2722{
Daniel Vetter23010e42010-03-08 13:35:02 +01002723 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002724 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08002725 int ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002726
Eric Anholt02354392008-11-26 13:58:13 -08002727 /* Not valid to be called on unbound objects. */
2728 if (obj_priv->gtt_space == NULL)
2729 return -EINVAL;
2730
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002731 ret = i915_gem_object_flush_gpu_write_domain(obj);
2732 if (ret != 0)
2733 return ret;
2734
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002735 /* Wait on any GPU rendering and flushing to occur. */
Daniel Vettere35a41d2010-02-11 22:13:59 +01002736 ret = i915_gem_object_wait_rendering(obj, true);
Eric Anholte47c68e2008-11-14 13:35:19 -08002737 if (ret != 0)
2738 return ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002739
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002740 old_write_domain = obj->write_domain;
2741 old_read_domains = obj->read_domains;
2742
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002743 /* If we're writing through the GTT domain, then CPU and GPU caches
2744 * will need to be invalidated at next use.
2745 */
2746 if (write)
Eric Anholte47c68e2008-11-14 13:35:19 -08002747 obj->read_domains &= I915_GEM_DOMAIN_GTT;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002748
Eric Anholte47c68e2008-11-14 13:35:19 -08002749 i915_gem_object_flush_cpu_write_domain(obj);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002750
2751 /* It should now be out of any other write domains, and we can update
2752 * the domain values for our changes.
2753 */
2754 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2755 obj->read_domains |= I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08002756 if (write) {
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002757 obj->write_domain = I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08002758 obj_priv->dirty = 1;
2759 }
2760
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002761 trace_i915_gem_object_change_domain(obj,
2762 old_read_domains,
2763 old_write_domain);
2764
Eric Anholte47c68e2008-11-14 13:35:19 -08002765 return 0;
2766}
2767
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002768/*
2769 * Prepare buffer for display plane. Use uninterruptible for possible flush
2770 * wait, as in modesetting process we're not supposed to be interrupted.
2771 */
2772int
2773i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2774{
Daniel Vetter23010e42010-03-08 13:35:02 +01002775 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002776 uint32_t old_write_domain, old_read_domains;
2777 int ret;
2778
2779 /* Not valid to be called on unbound objects. */
2780 if (obj_priv->gtt_space == NULL)
2781 return -EINVAL;
2782
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002783 ret = i915_gem_object_flush_gpu_write_domain(obj);
2784 if (ret)
2785 return ret;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002786
2787 /* Wait on any GPU rendering and flushing to occur. */
Daniel Vettere35a41d2010-02-11 22:13:59 +01002788 ret = i915_gem_object_wait_rendering(obj, false);
2789 if (ret != 0)
2790 return ret;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002791
Chris Wilsonb118c1e2010-05-27 13:18:14 +01002792 i915_gem_object_flush_cpu_write_domain(obj);
2793
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002794 old_write_domain = obj->write_domain;
2795 old_read_domains = obj->read_domains;
2796
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002797 /* It should now be out of any other write domains, and we can update
2798 * the domain values for our changes.
2799 */
2800 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
Chris Wilsonb118c1e2010-05-27 13:18:14 +01002801 obj->read_domains = I915_GEM_DOMAIN_GTT;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002802 obj->write_domain = I915_GEM_DOMAIN_GTT;
2803 obj_priv->dirty = 1;
2804
2805 trace_i915_gem_object_change_domain(obj,
2806 old_read_domains,
2807 old_write_domain);
2808
2809 return 0;
2810}
2811
Eric Anholte47c68e2008-11-14 13:35:19 -08002812/**
2813 * Moves a single object to the CPU read, and possibly write domain.
2814 *
2815 * This function returns when the move is complete, including waiting on
2816 * flushes to occur.
2817 */
2818static int
2819i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2820{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002821 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08002822 int ret;
2823
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002824 ret = i915_gem_object_flush_gpu_write_domain(obj);
2825 if (ret)
2826 return ret;
2827
Eric Anholte47c68e2008-11-14 13:35:19 -08002828 /* Wait on any GPU rendering and flushing to occur. */
Daniel Vettere35a41d2010-02-11 22:13:59 +01002829 ret = i915_gem_object_wait_rendering(obj, true);
Eric Anholte47c68e2008-11-14 13:35:19 -08002830 if (ret != 0)
2831 return ret;
2832
2833 i915_gem_object_flush_gtt_write_domain(obj);
2834
2835 /* If we have a partially-valid cache of the object in the CPU,
2836 * finish invalidating it and free the per-page flags.
2837 */
2838 i915_gem_object_set_to_full_cpu_read_domain(obj);
2839
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002840 old_write_domain = obj->write_domain;
2841 old_read_domains = obj->read_domains;
2842
Eric Anholte47c68e2008-11-14 13:35:19 -08002843 /* Flush the CPU cache if it's still invalid. */
2844 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2845 i915_gem_clflush_object(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08002846
2847 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2848 }
2849
2850 /* It should now be out of any other write domains, and we can update
2851 * the domain values for our changes.
2852 */
2853 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2854
2855 /* If we're writing through the CPU, then the GPU read domains will
2856 * need to be invalidated at next use.
2857 */
2858 if (write) {
2859 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2860 obj->write_domain = I915_GEM_DOMAIN_CPU;
2861 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002862
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002863 trace_i915_gem_object_change_domain(obj,
2864 old_read_domains,
2865 old_write_domain);
2866
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002867 return 0;
2868}
2869
Eric Anholt673a3942008-07-30 12:06:12 -07002870/*
2871 * Set the next domain for the specified object. This
2872 * may not actually perform the necessary flushing/invaliding though,
2873 * as that may want to be batched with other set_domain operations
2874 *
2875 * This is (we hope) the only really tricky part of gem. The goal
2876 * is fairly simple -- track which caches hold bits of the object
2877 * and make sure they remain coherent. A few concrete examples may
2878 * help to explain how it works. For shorthand, we use the notation
2879 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2880 * a pair of read and write domain masks.
2881 *
2882 * Case 1: the batch buffer
2883 *
2884 * 1. Allocated
2885 * 2. Written by CPU
2886 * 3. Mapped to GTT
2887 * 4. Read by GPU
2888 * 5. Unmapped from GTT
2889 * 6. Freed
2890 *
2891 * Let's take these a step at a time
2892 *
2893 * 1. Allocated
2894 * Pages allocated from the kernel may still have
2895 * cache contents, so we set them to (CPU, CPU) always.
2896 * 2. Written by CPU (using pwrite)
2897 * The pwrite function calls set_domain (CPU, CPU) and
2898 * this function does nothing (as nothing changes)
2899 * 3. Mapped by GTT
2900 * This function asserts that the object is not
2901 * currently in any GPU-based read or write domains
2902 * 4. Read by GPU
2903 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2904 * As write_domain is zero, this function adds in the
2905 * current read domains (CPU+COMMAND, 0).
2906 * flush_domains is set to CPU.
2907 * invalidate_domains is set to COMMAND
2908 * clflush is run to get data out of the CPU caches
2909 * then i915_dev_set_domain calls i915_gem_flush to
2910 * emit an MI_FLUSH and drm_agp_chipset_flush
2911 * 5. Unmapped from GTT
2912 * i915_gem_object_unbind calls set_domain (CPU, CPU)
2913 * flush_domains and invalidate_domains end up both zero
2914 * so no flushing/invalidating happens
2915 * 6. Freed
2916 * yay, done
2917 *
2918 * Case 2: The shared render buffer
2919 *
2920 * 1. Allocated
2921 * 2. Mapped to GTT
2922 * 3. Read/written by GPU
2923 * 4. set_domain to (CPU,CPU)
2924 * 5. Read/written by CPU
2925 * 6. Read/written by GPU
2926 *
2927 * 1. Allocated
2928 * Same as last example, (CPU, CPU)
2929 * 2. Mapped to GTT
2930 * Nothing changes (assertions find that it is not in the GPU)
2931 * 3. Read/written by GPU
2932 * execbuffer calls set_domain (RENDER, RENDER)
2933 * flush_domains gets CPU
2934 * invalidate_domains gets GPU
2935 * clflush (obj)
2936 * MI_FLUSH and drm_agp_chipset_flush
2937 * 4. set_domain (CPU, CPU)
2938 * flush_domains gets GPU
2939 * invalidate_domains gets CPU
2940 * wait_rendering (obj) to make sure all drawing is complete.
2941 * This will include an MI_FLUSH to get the data from GPU
2942 * to memory
2943 * clflush (obj) to invalidate the CPU cache
2944 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2945 * 5. Read/written by CPU
2946 * cache lines are loaded and dirtied
2947 * 6. Read written by GPU
2948 * Same as last GPU access
2949 *
2950 * Case 3: The constant buffer
2951 *
2952 * 1. Allocated
2953 * 2. Written by CPU
2954 * 3. Read by GPU
2955 * 4. Updated (written) by CPU again
2956 * 5. Read by GPU
2957 *
2958 * 1. Allocated
2959 * (CPU, CPU)
2960 * 2. Written by CPU
2961 * (CPU, CPU)
2962 * 3. Read by GPU
2963 * (CPU+RENDER, 0)
2964 * flush_domains = CPU
2965 * invalidate_domains = RENDER
2966 * clflush (obj)
2967 * MI_FLUSH
2968 * drm_agp_chipset_flush
2969 * 4. Updated (written) by CPU again
2970 * (CPU, CPU)
2971 * flush_domains = 0 (no previous write domain)
2972 * invalidate_domains = 0 (no new read domains)
2973 * 5. Read by GPU
2974 * (CPU+RENDER, 0)
2975 * flush_domains = CPU
2976 * invalidate_domains = RENDER
2977 * clflush (obj)
2978 * MI_FLUSH
2979 * drm_agp_chipset_flush
2980 */
Keith Packardc0d90822008-11-20 23:11:08 -08002981static void
Eric Anholt8b0e3782009-02-19 14:40:50 -08002982i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07002983{
2984 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01002985 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002986 uint32_t invalidate_domains = 0;
2987 uint32_t flush_domains = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002988 uint32_t old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08002989
Eric Anholt8b0e3782009-02-19 14:40:50 -08002990 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
2991 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
Eric Anholt673a3942008-07-30 12:06:12 -07002992
Jesse Barnes652c3932009-08-17 13:31:43 -07002993 intel_mark_busy(dev, obj);
2994
Eric Anholt673a3942008-07-30 12:06:12 -07002995#if WATCH_BUF
2996 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
2997 __func__, obj,
Eric Anholt8b0e3782009-02-19 14:40:50 -08002998 obj->read_domains, obj->pending_read_domains,
2999 obj->write_domain, obj->pending_write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07003000#endif
3001 /*
3002 * If the object isn't moving to a new write domain,
3003 * let the object stay in multiple read domains
3004 */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003005 if (obj->pending_write_domain == 0)
3006 obj->pending_read_domains |= obj->read_domains;
Eric Anholt673a3942008-07-30 12:06:12 -07003007 else
3008 obj_priv->dirty = 1;
3009
3010 /*
3011 * Flush the current write domain if
3012 * the new read domains don't match. Invalidate
3013 * any read domains which differ from the old
3014 * write domain
3015 */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003016 if (obj->write_domain &&
3017 obj->write_domain != obj->pending_read_domains) {
Eric Anholt673a3942008-07-30 12:06:12 -07003018 flush_domains |= obj->write_domain;
Eric Anholt8b0e3782009-02-19 14:40:50 -08003019 invalidate_domains |=
3020 obj->pending_read_domains & ~obj->write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -07003021 }
3022 /*
3023 * Invalidate any read caches which may have
3024 * stale data. That is, any new read domains.
3025 */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003026 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
Eric Anholt673a3942008-07-30 12:06:12 -07003027 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3028#if WATCH_BUF
3029 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3030 __func__, flush_domains, invalidate_domains);
3031#endif
Eric Anholt673a3942008-07-30 12:06:12 -07003032 i915_gem_clflush_object(obj);
3033 }
3034
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003035 old_read_domains = obj->read_domains;
3036
Eric Anholtefbeed92009-02-19 14:54:51 -08003037 /* The actual obj->write_domain will be updated with
3038 * pending_write_domain after we emit the accumulated flush for all
3039 * of our domain changes in execbuffers (which clears objects'
3040 * write_domains). So if we have a current write domain that we
3041 * aren't changing, set pending_write_domain to that.
3042 */
3043 if (flush_domains == 0 && obj->pending_write_domain == 0)
3044 obj->pending_write_domain = obj->write_domain;
Eric Anholt8b0e3782009-02-19 14:40:50 -08003045 obj->read_domains = obj->pending_read_domains;
Eric Anholt673a3942008-07-30 12:06:12 -07003046
3047 dev->invalidate_domains |= invalidate_domains;
3048 dev->flush_domains |= flush_domains;
3049#if WATCH_BUF
3050 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3051 __func__,
3052 obj->read_domains, obj->write_domain,
3053 dev->invalidate_domains, dev->flush_domains);
3054#endif
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003055
3056 trace_i915_gem_object_change_domain(obj,
3057 old_read_domains,
3058 obj->write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07003059}
3060
3061/**
Eric Anholte47c68e2008-11-14 13:35:19 -08003062 * Moves the object from a partially CPU read to a full one.
Eric Anholt673a3942008-07-30 12:06:12 -07003063 *
Eric Anholte47c68e2008-11-14 13:35:19 -08003064 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3065 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3066 */
3067static void
3068i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3069{
Daniel Vetter23010e42010-03-08 13:35:02 +01003070 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08003071
3072 if (!obj_priv->page_cpu_valid)
3073 return;
3074
3075 /* If we're partially in the CPU read domain, finish moving it in.
3076 */
3077 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3078 int i;
3079
3080 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3081 if (obj_priv->page_cpu_valid[i])
3082 continue;
Eric Anholt856fa192009-03-19 14:10:50 -07003083 drm_clflush_pages(obj_priv->pages + i, 1);
Eric Anholte47c68e2008-11-14 13:35:19 -08003084 }
Eric Anholte47c68e2008-11-14 13:35:19 -08003085 }
3086
3087 /* Free the page_cpu_valid mappings which are now stale, whether
3088 * or not we've got I915_GEM_DOMAIN_CPU.
3089 */
Eric Anholt9a298b22009-03-24 12:23:04 -07003090 kfree(obj_priv->page_cpu_valid);
Eric Anholte47c68e2008-11-14 13:35:19 -08003091 obj_priv->page_cpu_valid = NULL;
3092}
3093
3094/**
3095 * Set the CPU read domain on a range of the object.
3096 *
3097 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3098 * not entirely valid. The page_cpu_valid member of the object flags which
3099 * pages have been flushed, and will be respected by
3100 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3101 * of the whole object.
3102 *
3103 * This function returns when the move is complete, including waiting on
3104 * flushes to occur.
Eric Anholt673a3942008-07-30 12:06:12 -07003105 */
3106static int
Eric Anholte47c68e2008-11-14 13:35:19 -08003107i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3108 uint64_t offset, uint64_t size)
Eric Anholt673a3942008-07-30 12:06:12 -07003109{
Daniel Vetter23010e42010-03-08 13:35:02 +01003110 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003111 uint32_t old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003112 int i, ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003113
Eric Anholte47c68e2008-11-14 13:35:19 -08003114 if (offset == 0 && size == obj->size)
3115 return i915_gem_object_set_to_cpu_domain(obj, 0);
3116
Chris Wilson2dafb1e2010-06-07 14:03:05 +01003117 ret = i915_gem_object_flush_gpu_write_domain(obj);
3118 if (ret)
3119 return ret;
3120
Eric Anholte47c68e2008-11-14 13:35:19 -08003121 /* Wait on any GPU rendering and flushing to occur. */
Daniel Vettere35a41d2010-02-11 22:13:59 +01003122 ret = i915_gem_object_wait_rendering(obj, true);
Eric Anholte47c68e2008-11-14 13:35:19 -08003123 if (ret != 0)
3124 return ret;
3125 i915_gem_object_flush_gtt_write_domain(obj);
3126
3127 /* If we're already fully in the CPU read domain, we're done. */
3128 if (obj_priv->page_cpu_valid == NULL &&
3129 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07003130 return 0;
3131
Eric Anholte47c68e2008-11-14 13:35:19 -08003132 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3133 * newly adding I915_GEM_DOMAIN_CPU
3134 */
Eric Anholt673a3942008-07-30 12:06:12 -07003135 if (obj_priv->page_cpu_valid == NULL) {
Eric Anholt9a298b22009-03-24 12:23:04 -07003136 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3137 GFP_KERNEL);
Eric Anholte47c68e2008-11-14 13:35:19 -08003138 if (obj_priv->page_cpu_valid == NULL)
3139 return -ENOMEM;
3140 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3141 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
Eric Anholt673a3942008-07-30 12:06:12 -07003142
3143 /* Flush the cache on any pages that are still invalid from the CPU's
3144 * perspective.
3145 */
Eric Anholte47c68e2008-11-14 13:35:19 -08003146 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3147 i++) {
Eric Anholt673a3942008-07-30 12:06:12 -07003148 if (obj_priv->page_cpu_valid[i])
3149 continue;
3150
Eric Anholt856fa192009-03-19 14:10:50 -07003151 drm_clflush_pages(obj_priv->pages + i, 1);
Eric Anholt673a3942008-07-30 12:06:12 -07003152
3153 obj_priv->page_cpu_valid[i] = 1;
3154 }
3155
Eric Anholte47c68e2008-11-14 13:35:19 -08003156 /* It should now be out of any other write domains, and we can update
3157 * the domain values for our changes.
3158 */
3159 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3160
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003161 old_read_domains = obj->read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003162 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3163
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003164 trace_i915_gem_object_change_domain(obj,
3165 old_read_domains,
3166 obj->write_domain);
3167
Eric Anholt673a3942008-07-30 12:06:12 -07003168 return 0;
3169}
3170
3171/**
Eric Anholt673a3942008-07-30 12:06:12 -07003172 * Pin an object to the GTT and evaluate the relocations landing in it.
3173 */
3174static int
3175i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3176 struct drm_file *file_priv,
Jesse Barnes76446ca2009-12-17 22:05:42 -05003177 struct drm_i915_gem_exec_object2 *entry,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003178 struct drm_i915_gem_relocation_entry *relocs)
Eric Anholt673a3942008-07-30 12:06:12 -07003179{
3180 struct drm_device *dev = obj->dev;
Keith Packard0839ccb2008-10-30 19:38:48 -07003181 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01003182 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003183 int i, ret;
Keith Packard0839ccb2008-10-30 19:38:48 -07003184 void __iomem *reloc_page;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003185 bool need_fence;
3186
3187 need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3188 obj_priv->tiling_mode != I915_TILING_NONE;
3189
3190 /* Check fence reg constraints and rebind if necessary */
Chris Wilson808b24d2010-05-27 13:18:15 +01003191 if (need_fence &&
3192 !i915_gem_object_fence_offset_ok(obj,
3193 obj_priv->tiling_mode)) {
3194 ret = i915_gem_object_unbind(obj);
3195 if (ret)
3196 return ret;
3197 }
Eric Anholt673a3942008-07-30 12:06:12 -07003198
3199 /* Choose the GTT offset for our buffer and put it there. */
3200 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3201 if (ret)
3202 return ret;
3203
Jesse Barnes76446ca2009-12-17 22:05:42 -05003204 /*
3205 * Pre-965 chips need a fence register set up in order to
3206 * properly handle blits to/from tiled surfaces.
3207 */
3208 if (need_fence) {
3209 ret = i915_gem_object_get_fence_reg(obj);
3210 if (ret != 0) {
Jesse Barnes76446ca2009-12-17 22:05:42 -05003211 i915_gem_object_unpin(obj);
3212 return ret;
3213 }
3214 }
3215
Eric Anholt673a3942008-07-30 12:06:12 -07003216 entry->offset = obj_priv->gtt_offset;
3217
Eric Anholt673a3942008-07-30 12:06:12 -07003218 /* Apply the relocations, using the GTT aperture to avoid cache
3219 * flushing requirements.
3220 */
3221 for (i = 0; i < entry->relocation_count; i++) {
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003222 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
Eric Anholt673a3942008-07-30 12:06:12 -07003223 struct drm_gem_object *target_obj;
3224 struct drm_i915_gem_object *target_obj_priv;
Eric Anholt3043c602008-10-02 12:24:47 -07003225 uint32_t reloc_val, reloc_offset;
3226 uint32_t __iomem *reloc_entry;
Eric Anholt673a3942008-07-30 12:06:12 -07003227
Eric Anholt673a3942008-07-30 12:06:12 -07003228 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003229 reloc->target_handle);
Eric Anholt673a3942008-07-30 12:06:12 -07003230 if (target_obj == NULL) {
3231 i915_gem_object_unpin(obj);
Chris Wilsonbf79cb92010-08-04 14:19:46 +01003232 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07003233 }
Daniel Vetter23010e42010-03-08 13:35:02 +01003234 target_obj_priv = to_intel_bo(target_obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003235
Chris Wilson8542a0b2009-09-09 21:15:15 +01003236#if WATCH_RELOC
3237 DRM_INFO("%s: obj %p offset %08x target %d "
3238 "read %08x write %08x gtt %08x "
3239 "presumed %08x delta %08x\n",
3240 __func__,
3241 obj,
3242 (int) reloc->offset,
3243 (int) reloc->target_handle,
3244 (int) reloc->read_domains,
3245 (int) reloc->write_domain,
3246 (int) target_obj_priv->gtt_offset,
3247 (int) reloc->presumed_offset,
3248 reloc->delta);
3249#endif
3250
Eric Anholt673a3942008-07-30 12:06:12 -07003251 /* The target buffer should have appeared before us in the
3252 * exec_object list, so it should have a GTT space bound by now.
3253 */
3254 if (target_obj_priv->gtt_space == NULL) {
3255 DRM_ERROR("No GTT space found for object %d\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003256 reloc->target_handle);
Eric Anholt673a3942008-07-30 12:06:12 -07003257 drm_gem_object_unreference(target_obj);
3258 i915_gem_object_unpin(obj);
3259 return -EINVAL;
3260 }
3261
Chris Wilson8542a0b2009-09-09 21:15:15 +01003262 /* Validate that the target is in a valid r/w GPU domain */
Daniel Vetter16edd552010-02-19 11:52:02 +01003263 if (reloc->write_domain & (reloc->write_domain - 1)) {
3264 DRM_ERROR("reloc with multiple write domains: "
3265 "obj %p target %d offset %d "
3266 "read %08x write %08x",
3267 obj, reloc->target_handle,
3268 (int) reloc->offset,
3269 reloc->read_domains,
3270 reloc->write_domain);
3271 return -EINVAL;
3272 }
Chris Wilson8542a0b2009-09-09 21:15:15 +01003273 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3274 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3275 DRM_ERROR("reloc with read/write CPU domains: "
3276 "obj %p target %d offset %d "
3277 "read %08x write %08x",
3278 obj, reloc->target_handle,
3279 (int) reloc->offset,
3280 reloc->read_domains,
3281 reloc->write_domain);
3282 drm_gem_object_unreference(target_obj);
3283 i915_gem_object_unpin(obj);
3284 return -EINVAL;
3285 }
3286 if (reloc->write_domain && target_obj->pending_write_domain &&
3287 reloc->write_domain != target_obj->pending_write_domain) {
3288 DRM_ERROR("Write domain conflict: "
3289 "obj %p target %d offset %d "
3290 "new %08x old %08x\n",
3291 obj, reloc->target_handle,
3292 (int) reloc->offset,
3293 reloc->write_domain,
3294 target_obj->pending_write_domain);
3295 drm_gem_object_unreference(target_obj);
3296 i915_gem_object_unpin(obj);
3297 return -EINVAL;
3298 }
3299
3300 target_obj->pending_read_domains |= reloc->read_domains;
3301 target_obj->pending_write_domain |= reloc->write_domain;
3302
3303 /* If the relocation already has the right value in it, no
3304 * more work needs to be done.
3305 */
3306 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3307 drm_gem_object_unreference(target_obj);
3308 continue;
3309 }
3310
3311 /* Check that the relocation address is valid... */
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003312 if (reloc->offset > obj->size - 4) {
Eric Anholt673a3942008-07-30 12:06:12 -07003313 DRM_ERROR("Relocation beyond object bounds: "
3314 "obj %p target %d offset %d size %d.\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003315 obj, reloc->target_handle,
3316 (int) reloc->offset, (int) obj->size);
Eric Anholt673a3942008-07-30 12:06:12 -07003317 drm_gem_object_unreference(target_obj);
3318 i915_gem_object_unpin(obj);
3319 return -EINVAL;
3320 }
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003321 if (reloc->offset & 3) {
Eric Anholt673a3942008-07-30 12:06:12 -07003322 DRM_ERROR("Relocation not 4-byte aligned: "
3323 "obj %p target %d offset %d.\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003324 obj, reloc->target_handle,
3325 (int) reloc->offset);
Eric Anholt673a3942008-07-30 12:06:12 -07003326 drm_gem_object_unreference(target_obj);
3327 i915_gem_object_unpin(obj);
3328 return -EINVAL;
3329 }
3330
Chris Wilson8542a0b2009-09-09 21:15:15 +01003331 /* and points to somewhere within the target object. */
Chris Wilsoncd0b9fb2009-09-15 23:23:18 +01003332 if (reloc->delta >= target_obj->size) {
3333 DRM_ERROR("Relocation beyond target object bounds: "
3334 "obj %p target %d delta %d size %d.\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003335 obj, reloc->target_handle,
Chris Wilsoncd0b9fb2009-09-15 23:23:18 +01003336 (int) reloc->delta, (int) target_obj->size);
Chris Wilson491152b2009-02-11 14:26:32 +00003337 drm_gem_object_unreference(target_obj);
3338 i915_gem_object_unpin(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08003339 return -EINVAL;
3340 }
3341
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003342 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3343 if (ret != 0) {
3344 drm_gem_object_unreference(target_obj);
3345 i915_gem_object_unpin(obj);
3346 return -EINVAL;
Eric Anholt673a3942008-07-30 12:06:12 -07003347 }
3348
3349 /* Map the page containing the relocation we're going to
3350 * perform.
3351 */
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003352 reloc_offset = obj_priv->gtt_offset + reloc->offset;
Keith Packard0839ccb2008-10-30 19:38:48 -07003353 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3354 (reloc_offset &
Chris Wilsonfca3ec02010-08-04 14:34:24 +01003355 ~(PAGE_SIZE - 1)),
3356 KM_USER0);
Eric Anholt3043c602008-10-02 12:24:47 -07003357 reloc_entry = (uint32_t __iomem *)(reloc_page +
Keith Packard0839ccb2008-10-30 19:38:48 -07003358 (reloc_offset & (PAGE_SIZE - 1)));
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003359 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
Eric Anholt673a3942008-07-30 12:06:12 -07003360
3361#if WATCH_BUF
3362 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003363 obj, (unsigned int) reloc->offset,
Eric Anholt673a3942008-07-30 12:06:12 -07003364 readl(reloc_entry), reloc_val);
3365#endif
3366 writel(reloc_val, reloc_entry);
Chris Wilsonfca3ec02010-08-04 14:34:24 +01003367 io_mapping_unmap_atomic(reloc_page, KM_USER0);
Eric Anholt673a3942008-07-30 12:06:12 -07003368
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003369 /* The updated presumed offset for this entry will be
3370 * copied back out to the user.
Eric Anholt673a3942008-07-30 12:06:12 -07003371 */
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003372 reloc->presumed_offset = target_obj_priv->gtt_offset;
Eric Anholt673a3942008-07-30 12:06:12 -07003373
3374 drm_gem_object_unreference(target_obj);
3375 }
3376
Eric Anholt673a3942008-07-30 12:06:12 -07003377#if WATCH_BUF
3378 if (0)
3379 i915_gem_dump_object(obj, 128, __func__, ~0);
3380#endif
3381 return 0;
3382}
3383
Eric Anholt673a3942008-07-30 12:06:12 -07003384/* Throttle our rendering by waiting until the ring has completed our requests
3385 * emitted over 20 msec ago.
3386 *
Eric Anholtb9624422009-06-03 07:27:35 +00003387 * Note that if we were to use the current jiffies each time around the loop,
3388 * we wouldn't escape the function with any frames outstanding if the time to
3389 * render a frame was over 20ms.
3390 *
Eric Anholt673a3942008-07-30 12:06:12 -07003391 * This should get us reasonable parallelism between CPU and GPU but also
3392 * relatively low latency when blocking on a particular request to finish.
3393 */
3394static int
3395i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3396{
3397 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3398 int ret = 0;
Eric Anholtb9624422009-06-03 07:27:35 +00003399 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
Eric Anholt673a3942008-07-30 12:06:12 -07003400
3401 mutex_lock(&dev->struct_mutex);
Eric Anholtb9624422009-06-03 07:27:35 +00003402 while (!list_empty(&i915_file_priv->mm.request_list)) {
3403 struct drm_i915_gem_request *request;
3404
3405 request = list_first_entry(&i915_file_priv->mm.request_list,
3406 struct drm_i915_gem_request,
3407 client_list);
3408
3409 if (time_after_eq(request->emitted_jiffies, recent_enough))
3410 break;
3411
Zou Nan hai852835f2010-05-21 09:08:56 +08003412 ret = i915_wait_request(dev, request->seqno, request->ring);
Eric Anholtb9624422009-06-03 07:27:35 +00003413 if (ret != 0)
3414 break;
3415 }
Eric Anholt673a3942008-07-30 12:06:12 -07003416 mutex_unlock(&dev->struct_mutex);
Eric Anholtb9624422009-06-03 07:27:35 +00003417
Eric Anholt673a3942008-07-30 12:06:12 -07003418 return ret;
3419}
3420
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003421static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003422i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003423 uint32_t buffer_count,
3424 struct drm_i915_gem_relocation_entry **relocs)
3425{
3426 uint32_t reloc_count = 0, reloc_index = 0, i;
3427 int ret;
3428
3429 *relocs = NULL;
3430 for (i = 0; i < buffer_count; i++) {
3431 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3432 return -EINVAL;
3433 reloc_count += exec_list[i].relocation_count;
3434 }
3435
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003436 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
Jesse Barnes76446ca2009-12-17 22:05:42 -05003437 if (*relocs == NULL) {
3438 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003439 return -ENOMEM;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003440 }
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003441
3442 for (i = 0; i < buffer_count; i++) {
3443 struct drm_i915_gem_relocation_entry __user *user_relocs;
3444
3445 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3446
3447 ret = copy_from_user(&(*relocs)[reloc_index],
3448 user_relocs,
3449 exec_list[i].relocation_count *
3450 sizeof(**relocs));
3451 if (ret != 0) {
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003452 drm_free_large(*relocs);
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003453 *relocs = NULL;
Florian Mickler2bc43b52009-04-06 22:55:41 +02003454 return -EFAULT;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003455 }
3456
3457 reloc_index += exec_list[i].relocation_count;
3458 }
3459
Florian Mickler2bc43b52009-04-06 22:55:41 +02003460 return 0;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003461}
3462
3463static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003464i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003465 uint32_t buffer_count,
3466 struct drm_i915_gem_relocation_entry *relocs)
3467{
3468 uint32_t reloc_count = 0, i;
Florian Mickler2bc43b52009-04-06 22:55:41 +02003469 int ret = 0;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003470
Chris Wilson93533c22010-01-31 10:40:48 +00003471 if (relocs == NULL)
3472 return 0;
3473
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003474 for (i = 0; i < buffer_count; i++) {
3475 struct drm_i915_gem_relocation_entry __user *user_relocs;
Florian Mickler2bc43b52009-04-06 22:55:41 +02003476 int unwritten;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003477
3478 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3479
Florian Mickler2bc43b52009-04-06 22:55:41 +02003480 unwritten = copy_to_user(user_relocs,
3481 &relocs[reloc_count],
3482 exec_list[i].relocation_count *
3483 sizeof(*relocs));
3484
3485 if (unwritten) {
3486 ret = -EFAULT;
3487 goto err;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003488 }
3489
3490 reloc_count += exec_list[i].relocation_count;
3491 }
3492
Florian Mickler2bc43b52009-04-06 22:55:41 +02003493err:
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003494 drm_free_large(relocs);
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003495
3496 return ret;
3497}
3498
Chris Wilson83d60792009-06-06 09:45:57 +01003499static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003500i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
Chris Wilson83d60792009-06-06 09:45:57 +01003501 uint64_t exec_offset)
3502{
3503 uint32_t exec_start, exec_len;
3504
3505 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3506 exec_len = (uint32_t) exec->batch_len;
3507
3508 if ((exec_start | exec_len) & 0x7)
3509 return -EINVAL;
3510
3511 if (!exec_start)
3512 return -EINVAL;
3513
3514 return 0;
3515}
3516
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003517static int
3518i915_gem_wait_for_pending_flip(struct drm_device *dev,
3519 struct drm_gem_object **object_list,
3520 int count)
3521{
3522 drm_i915_private_t *dev_priv = dev->dev_private;
3523 struct drm_i915_gem_object *obj_priv;
3524 DEFINE_WAIT(wait);
3525 int i, ret = 0;
3526
3527 for (;;) {
3528 prepare_to_wait(&dev_priv->pending_flip_queue,
3529 &wait, TASK_INTERRUPTIBLE);
3530 for (i = 0; i < count; i++) {
Daniel Vetter23010e42010-03-08 13:35:02 +01003531 obj_priv = to_intel_bo(object_list[i]);
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003532 if (atomic_read(&obj_priv->pending_flip) > 0)
3533 break;
3534 }
3535 if (i == count)
3536 break;
3537
3538 if (!signal_pending(current)) {
3539 mutex_unlock(&dev->struct_mutex);
3540 schedule();
3541 mutex_lock(&dev->struct_mutex);
3542 continue;
3543 }
3544 ret = -ERESTARTSYS;
3545 break;
3546 }
3547 finish_wait(&dev_priv->pending_flip_queue, &wait);
3548
3549 return ret;
3550}
3551
Chris Wilson43b27f42010-07-02 08:57:15 +01003552
Eric Anholt673a3942008-07-30 12:06:12 -07003553int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003554i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3555 struct drm_file *file_priv,
3556 struct drm_i915_gem_execbuffer2 *args,
3557 struct drm_i915_gem_exec_object2 *exec_list)
Eric Anholt673a3942008-07-30 12:06:12 -07003558{
3559 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07003560 struct drm_gem_object **object_list = NULL;
3561 struct drm_gem_object *batch_obj;
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003562 struct drm_i915_gem_object *obj_priv;
Eric Anholt201361a2009-03-11 12:30:04 -07003563 struct drm_clip_rect *cliprects = NULL;
Chris Wilson93533c22010-01-31 10:40:48 +00003564 struct drm_i915_gem_relocation_entry *relocs = NULL;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003565 int ret = 0, ret2, i, pinned = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07003566 uint64_t exec_offset;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003567 uint32_t seqno, flush_domains, reloc_index;
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003568 int pin_tries, flips;
Eric Anholt673a3942008-07-30 12:06:12 -07003569
Zou Nan hai852835f2010-05-21 09:08:56 +08003570 struct intel_ring_buffer *ring = NULL;
3571
Eric Anholt673a3942008-07-30 12:06:12 -07003572#if WATCH_EXEC
3573 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3574 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3575#endif
Zou Nan haid1b851f2010-05-21 09:08:57 +08003576 if (args->flags & I915_EXEC_BSD) {
3577 if (!HAS_BSD(dev)) {
3578 DRM_ERROR("execbuf with wrong flag\n");
3579 return -EINVAL;
3580 }
3581 ring = &dev_priv->bsd_ring;
3582 } else {
3583 ring = &dev_priv->render_ring;
3584 }
3585
Eric Anholt4f481ed2008-09-10 14:22:49 -07003586 if (args->buffer_count < 1) {
3587 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3588 return -EINVAL;
3589 }
Eric Anholtc8e0f932009-11-22 03:49:37 +01003590 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
Jesse Barnes76446ca2009-12-17 22:05:42 -05003591 if (object_list == NULL) {
3592 DRM_ERROR("Failed to allocate object list for %d buffers\n",
Eric Anholt673a3942008-07-30 12:06:12 -07003593 args->buffer_count);
3594 ret = -ENOMEM;
3595 goto pre_mutex_err;
3596 }
Eric Anholt673a3942008-07-30 12:06:12 -07003597
Eric Anholt201361a2009-03-11 12:30:04 -07003598 if (args->num_cliprects != 0) {
Eric Anholt9a298b22009-03-24 12:23:04 -07003599 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3600 GFP_KERNEL);
Owain Ainswortha40e8d32010-02-09 14:25:55 +00003601 if (cliprects == NULL) {
3602 ret = -ENOMEM;
Eric Anholt201361a2009-03-11 12:30:04 -07003603 goto pre_mutex_err;
Owain Ainswortha40e8d32010-02-09 14:25:55 +00003604 }
Eric Anholt201361a2009-03-11 12:30:04 -07003605
3606 ret = copy_from_user(cliprects,
3607 (struct drm_clip_rect __user *)
3608 (uintptr_t) args->cliprects_ptr,
3609 sizeof(*cliprects) * args->num_cliprects);
3610 if (ret != 0) {
3611 DRM_ERROR("copy %d cliprects failed: %d\n",
3612 args->num_cliprects, ret);
Dan Carpenterc877cdc2010-06-23 19:03:01 +02003613 ret = -EFAULT;
Eric Anholt201361a2009-03-11 12:30:04 -07003614 goto pre_mutex_err;
3615 }
3616 }
3617
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003618 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3619 &relocs);
3620 if (ret != 0)
3621 goto pre_mutex_err;
3622
Eric Anholt673a3942008-07-30 12:06:12 -07003623 mutex_lock(&dev->struct_mutex);
3624
3625 i915_verify_inactive(dev, __FILE__, __LINE__);
3626
Ben Gamariba1234d2009-09-14 17:48:47 -04003627 if (atomic_read(&dev_priv->mm.wedged)) {
Eric Anholt673a3942008-07-30 12:06:12 -07003628 mutex_unlock(&dev->struct_mutex);
Chris Wilsona198bc82009-02-06 16:55:20 +00003629 ret = -EIO;
3630 goto pre_mutex_err;
Eric Anholt673a3942008-07-30 12:06:12 -07003631 }
3632
3633 if (dev_priv->mm.suspended) {
Eric Anholt673a3942008-07-30 12:06:12 -07003634 mutex_unlock(&dev->struct_mutex);
Chris Wilsona198bc82009-02-06 16:55:20 +00003635 ret = -EBUSY;
3636 goto pre_mutex_err;
Eric Anholt673a3942008-07-30 12:06:12 -07003637 }
3638
Keith Packardac94a962008-11-20 23:30:27 -08003639 /* Look up object handles */
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003640 flips = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07003641 for (i = 0; i < args->buffer_count; i++) {
3642 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3643 exec_list[i].handle);
3644 if (object_list[i] == NULL) {
3645 DRM_ERROR("Invalid object handle %d at index %d\n",
3646 exec_list[i].handle, i);
Chris Wilson0ce907f2010-01-23 20:26:35 +00003647 /* prevent error path from reading uninitialized data */
3648 args->buffer_count = i + 1;
Chris Wilsonbf79cb92010-08-04 14:19:46 +01003649 ret = -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07003650 goto err;
3651 }
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003652
Daniel Vetter23010e42010-03-08 13:35:02 +01003653 obj_priv = to_intel_bo(object_list[i]);
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003654 if (obj_priv->in_execbuffer) {
3655 DRM_ERROR("Object %p appears more than once in object list\n",
3656 object_list[i]);
Chris Wilson0ce907f2010-01-23 20:26:35 +00003657 /* prevent error path from reading uninitialized data */
3658 args->buffer_count = i + 1;
Chris Wilsonbf79cb92010-08-04 14:19:46 +01003659 ret = -EINVAL;
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003660 goto err;
3661 }
3662 obj_priv->in_execbuffer = true;
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003663 flips += atomic_read(&obj_priv->pending_flip);
3664 }
3665
3666 if (flips > 0) {
3667 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3668 args->buffer_count);
3669 if (ret)
3670 goto err;
Keith Packardac94a962008-11-20 23:30:27 -08003671 }
Eric Anholt673a3942008-07-30 12:06:12 -07003672
Keith Packardac94a962008-11-20 23:30:27 -08003673 /* Pin and relocate */
3674 for (pin_tries = 0; ; pin_tries++) {
3675 ret = 0;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003676 reloc_index = 0;
3677
Keith Packardac94a962008-11-20 23:30:27 -08003678 for (i = 0; i < args->buffer_count; i++) {
3679 object_list[i]->pending_read_domains = 0;
3680 object_list[i]->pending_write_domain = 0;
3681 ret = i915_gem_object_pin_and_relocate(object_list[i],
3682 file_priv,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003683 &exec_list[i],
3684 &relocs[reloc_index]);
Keith Packardac94a962008-11-20 23:30:27 -08003685 if (ret)
3686 break;
3687 pinned = i + 1;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003688 reloc_index += exec_list[i].relocation_count;
Keith Packardac94a962008-11-20 23:30:27 -08003689 }
3690 /* success */
3691 if (ret == 0)
3692 break;
3693
3694 /* error other than GTT full, or we've already tried again */
Chris Wilson2939e1f2009-06-06 09:46:03 +01003695 if (ret != -ENOSPC || pin_tries >= 1) {
Chris Wilson07f73f62009-09-14 16:50:30 +01003696 if (ret != -ERESTARTSYS) {
3697 unsigned long long total_size = 0;
Chris Wilson3d1cc472010-05-27 13:18:19 +01003698 int num_fences = 0;
3699 for (i = 0; i < args->buffer_count; i++) {
Chris Wilson43b27f42010-07-02 08:57:15 +01003700 obj_priv = to_intel_bo(object_list[i]);
Chris Wilson3d1cc472010-05-27 13:18:19 +01003701
Chris Wilson07f73f62009-09-14 16:50:30 +01003702 total_size += object_list[i]->size;
Chris Wilson3d1cc472010-05-27 13:18:19 +01003703 num_fences +=
3704 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3705 obj_priv->tiling_mode != I915_TILING_NONE;
3706 }
3707 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
Chris Wilson07f73f62009-09-14 16:50:30 +01003708 pinned+1, args->buffer_count,
Chris Wilson3d1cc472010-05-27 13:18:19 +01003709 total_size, num_fences,
3710 ret);
Chris Wilson07f73f62009-09-14 16:50:30 +01003711 DRM_ERROR("%d objects [%d pinned], "
3712 "%d object bytes [%d pinned], "
3713 "%d/%d gtt bytes\n",
3714 atomic_read(&dev->object_count),
3715 atomic_read(&dev->pin_count),
3716 atomic_read(&dev->object_memory),
3717 atomic_read(&dev->pin_memory),
3718 atomic_read(&dev->gtt_memory),
3719 dev->gtt_total);
3720 }
Eric Anholt673a3942008-07-30 12:06:12 -07003721 goto err;
3722 }
Keith Packardac94a962008-11-20 23:30:27 -08003723
3724 /* unpin all of our buffers */
3725 for (i = 0; i < pinned; i++)
3726 i915_gem_object_unpin(object_list[i]);
Eric Anholtb1177632008-12-10 10:09:41 -08003727 pinned = 0;
Keith Packardac94a962008-11-20 23:30:27 -08003728
3729 /* evict everyone we can from the aperture */
3730 ret = i915_gem_evict_everything(dev);
Chris Wilson07f73f62009-09-14 16:50:30 +01003731 if (ret && ret != -ENOSPC)
Keith Packardac94a962008-11-20 23:30:27 -08003732 goto err;
Eric Anholt673a3942008-07-30 12:06:12 -07003733 }
3734
3735 /* Set the pending read domains for the batch buffer to COMMAND */
3736 batch_obj = object_list[args->buffer_count-1];
Chris Wilson5f26a2c2009-06-06 09:45:58 +01003737 if (batch_obj->pending_write_domain) {
3738 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3739 ret = -EINVAL;
3740 goto err;
3741 }
3742 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
Eric Anholt673a3942008-07-30 12:06:12 -07003743
Chris Wilson83d60792009-06-06 09:45:57 +01003744 /* Sanity check the batch buffer, prior to moving objects */
3745 exec_offset = exec_list[args->buffer_count - 1].offset;
3746 ret = i915_gem_check_execbuffer (args, exec_offset);
3747 if (ret != 0) {
3748 DRM_ERROR("execbuf with invalid offset/length\n");
3749 goto err;
3750 }
3751
Eric Anholt673a3942008-07-30 12:06:12 -07003752 i915_verify_inactive(dev, __FILE__, __LINE__);
3753
Keith Packard646f0f62008-11-20 23:23:03 -08003754 /* Zero the global flush/invalidate flags. These
3755 * will be modified as new domains are computed
3756 * for each object
3757 */
3758 dev->invalidate_domains = 0;
3759 dev->flush_domains = 0;
3760
Eric Anholt673a3942008-07-30 12:06:12 -07003761 for (i = 0; i < args->buffer_count; i++) {
3762 struct drm_gem_object *obj = object_list[i];
Eric Anholt673a3942008-07-30 12:06:12 -07003763
Keith Packard646f0f62008-11-20 23:23:03 -08003764 /* Compute new gpu domains and update invalidate/flush */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003765 i915_gem_object_set_to_gpu_domain(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003766 }
3767
3768 i915_verify_inactive(dev, __FILE__, __LINE__);
3769
Keith Packard646f0f62008-11-20 23:23:03 -08003770 if (dev->invalidate_domains | dev->flush_domains) {
3771#if WATCH_EXEC
3772 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3773 __func__,
3774 dev->invalidate_domains,
3775 dev->flush_domains);
3776#endif
3777 i915_gem_flush(dev,
3778 dev->invalidate_domains,
3779 dev->flush_domains);
Daniel Vettera6910432010-02-02 17:08:37 +01003780 }
3781
3782 if (dev_priv->render_ring.outstanding_lazy_request) {
3783 (void)i915_add_request(dev, file_priv, 0,
3784 &dev_priv->render_ring);
3785 dev_priv->render_ring.outstanding_lazy_request = false;
3786 }
3787 if (dev_priv->bsd_ring.outstanding_lazy_request) {
3788 (void)i915_add_request(dev, file_priv, 0,
3789 &dev_priv->bsd_ring);
3790 dev_priv->bsd_ring.outstanding_lazy_request = false;
Keith Packard646f0f62008-11-20 23:23:03 -08003791 }
Eric Anholt673a3942008-07-30 12:06:12 -07003792
Eric Anholtefbeed92009-02-19 14:54:51 -08003793 for (i = 0; i < args->buffer_count; i++) {
3794 struct drm_gem_object *obj = object_list[i];
Daniel Vetter23010e42010-03-08 13:35:02 +01003795 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003796 uint32_t old_write_domain = obj->write_domain;
Eric Anholtefbeed92009-02-19 14:54:51 -08003797
3798 obj->write_domain = obj->pending_write_domain;
Daniel Vetter99fcb762010-02-07 16:20:18 +01003799 if (obj->write_domain)
3800 list_move_tail(&obj_priv->gpu_write_list,
3801 &dev_priv->mm.gpu_write_list);
3802 else
3803 list_del_init(&obj_priv->gpu_write_list);
3804
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003805 trace_i915_gem_object_change_domain(obj,
3806 obj->read_domains,
3807 old_write_domain);
Eric Anholtefbeed92009-02-19 14:54:51 -08003808 }
3809
Eric Anholt673a3942008-07-30 12:06:12 -07003810 i915_verify_inactive(dev, __FILE__, __LINE__);
3811
3812#if WATCH_COHERENCY
3813 for (i = 0; i < args->buffer_count; i++) {
3814 i915_gem_object_check_coherency(object_list[i],
3815 exec_list[i].handle);
3816 }
3817#endif
3818
Eric Anholt673a3942008-07-30 12:06:12 -07003819#if WATCH_EXEC
Ben Gamari6911a9b2009-04-02 11:24:54 -07003820 i915_gem_dump_object(batch_obj,
Eric Anholt673a3942008-07-30 12:06:12 -07003821 args->batch_len,
3822 __func__,
3823 ~0);
3824#endif
3825
Eric Anholt673a3942008-07-30 12:06:12 -07003826 /* Exec the batchbuffer */
Zou Nan hai852835f2010-05-21 09:08:56 +08003827 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3828 cliprects, exec_offset);
Eric Anholt673a3942008-07-30 12:06:12 -07003829 if (ret) {
3830 DRM_ERROR("dispatch failed %d\n", ret);
3831 goto err;
3832 }
3833
3834 /*
3835 * Ensure that the commands in the batch buffer are
3836 * finished before the interrupt fires
3837 */
Zou Nan hai852835f2010-05-21 09:08:56 +08003838 flush_domains = i915_retire_commands(dev, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07003839
3840 i915_verify_inactive(dev, __FILE__, __LINE__);
3841
3842 /*
3843 * Get a seqno representing the execution of the current buffer,
3844 * which we can wait on. We would like to mitigate these interrupts,
3845 * likely by only creating seqnos occasionally (so that we have
3846 * *some* interrupts representing completion of buffers that we can
3847 * wait on when trying to clear up gtt space).
3848 */
Zou Nan hai852835f2010-05-21 09:08:56 +08003849 seqno = i915_add_request(dev, file_priv, flush_domains, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07003850 BUG_ON(seqno == 0);
Eric Anholt673a3942008-07-30 12:06:12 -07003851 for (i = 0; i < args->buffer_count; i++) {
3852 struct drm_gem_object *obj = object_list[i];
Zou Nan hai852835f2010-05-21 09:08:56 +08003853 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003854
Zou Nan hai852835f2010-05-21 09:08:56 +08003855 i915_gem_object_move_to_active(obj, seqno, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07003856#if WATCH_LRU
3857 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3858#endif
3859 }
3860#if WATCH_LRU
3861 i915_dump_lru(dev, __func__);
3862#endif
3863
3864 i915_verify_inactive(dev, __FILE__, __LINE__);
3865
Eric Anholt673a3942008-07-30 12:06:12 -07003866err:
Julia Lawallaad87df2008-12-21 16:28:47 +01003867 for (i = 0; i < pinned; i++)
3868 i915_gem_object_unpin(object_list[i]);
Eric Anholt673a3942008-07-30 12:06:12 -07003869
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003870 for (i = 0; i < args->buffer_count; i++) {
3871 if (object_list[i]) {
Daniel Vetter23010e42010-03-08 13:35:02 +01003872 obj_priv = to_intel_bo(object_list[i]);
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003873 obj_priv->in_execbuffer = false;
3874 }
Julia Lawallaad87df2008-12-21 16:28:47 +01003875 drm_gem_object_unreference(object_list[i]);
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003876 }
Julia Lawallaad87df2008-12-21 16:28:47 +01003877
Eric Anholt673a3942008-07-30 12:06:12 -07003878 mutex_unlock(&dev->struct_mutex);
3879
Chris Wilson93533c22010-01-31 10:40:48 +00003880pre_mutex_err:
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003881 /* Copy the updated relocations out regardless of current error
3882 * state. Failure to update the relocs would mean that the next
3883 * time userland calls execbuf, it would do so with presumed offset
3884 * state that didn't match the actual object state.
3885 */
3886 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3887 relocs);
3888 if (ret2 != 0) {
3889 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3890
3891 if (ret == 0)
3892 ret = ret2;
3893 }
3894
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003895 drm_free_large(object_list);
Eric Anholt9a298b22009-03-24 12:23:04 -07003896 kfree(cliprects);
Eric Anholt673a3942008-07-30 12:06:12 -07003897
3898 return ret;
3899}
3900
Jesse Barnes76446ca2009-12-17 22:05:42 -05003901/*
3902 * Legacy execbuffer just creates an exec2 list from the original exec object
3903 * list array and passes it to the real function.
3904 */
3905int
3906i915_gem_execbuffer(struct drm_device *dev, void *data,
3907 struct drm_file *file_priv)
3908{
3909 struct drm_i915_gem_execbuffer *args = data;
3910 struct drm_i915_gem_execbuffer2 exec2;
3911 struct drm_i915_gem_exec_object *exec_list = NULL;
3912 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3913 int ret, i;
3914
3915#if WATCH_EXEC
3916 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3917 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3918#endif
3919
3920 if (args->buffer_count < 1) {
3921 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3922 return -EINVAL;
3923 }
3924
3925 /* Copy in the exec list from userland */
3926 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3927 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3928 if (exec_list == NULL || exec2_list == NULL) {
3929 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3930 args->buffer_count);
3931 drm_free_large(exec_list);
3932 drm_free_large(exec2_list);
3933 return -ENOMEM;
3934 }
3935 ret = copy_from_user(exec_list,
3936 (struct drm_i915_relocation_entry __user *)
3937 (uintptr_t) args->buffers_ptr,
3938 sizeof(*exec_list) * args->buffer_count);
3939 if (ret != 0) {
3940 DRM_ERROR("copy %d exec entries failed %d\n",
3941 args->buffer_count, ret);
3942 drm_free_large(exec_list);
3943 drm_free_large(exec2_list);
3944 return -EFAULT;
3945 }
3946
3947 for (i = 0; i < args->buffer_count; i++) {
3948 exec2_list[i].handle = exec_list[i].handle;
3949 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3950 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3951 exec2_list[i].alignment = exec_list[i].alignment;
3952 exec2_list[i].offset = exec_list[i].offset;
3953 if (!IS_I965G(dev))
3954 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3955 else
3956 exec2_list[i].flags = 0;
3957 }
3958
3959 exec2.buffers_ptr = args->buffers_ptr;
3960 exec2.buffer_count = args->buffer_count;
3961 exec2.batch_start_offset = args->batch_start_offset;
3962 exec2.batch_len = args->batch_len;
3963 exec2.DR1 = args->DR1;
3964 exec2.DR4 = args->DR4;
3965 exec2.num_cliprects = args->num_cliprects;
3966 exec2.cliprects_ptr = args->cliprects_ptr;
Zou Nan hai852835f2010-05-21 09:08:56 +08003967 exec2.flags = I915_EXEC_RENDER;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003968
3969 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3970 if (!ret) {
3971 /* Copy the new buffer offsets back to the user's exec list. */
3972 for (i = 0; i < args->buffer_count; i++)
3973 exec_list[i].offset = exec2_list[i].offset;
3974 /* ... and back out to userspace */
3975 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3976 (uintptr_t) args->buffers_ptr,
3977 exec_list,
3978 sizeof(*exec_list) * args->buffer_count);
3979 if (ret) {
3980 ret = -EFAULT;
3981 DRM_ERROR("failed to copy %d exec entries "
3982 "back to user (%d)\n",
3983 args->buffer_count, ret);
3984 }
Jesse Barnes76446ca2009-12-17 22:05:42 -05003985 }
3986
3987 drm_free_large(exec_list);
3988 drm_free_large(exec2_list);
3989 return ret;
3990}
3991
3992int
3993i915_gem_execbuffer2(struct drm_device *dev, void *data,
3994 struct drm_file *file_priv)
3995{
3996 struct drm_i915_gem_execbuffer2 *args = data;
3997 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3998 int ret;
3999
4000#if WATCH_EXEC
4001 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4002 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4003#endif
4004
4005 if (args->buffer_count < 1) {
4006 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4007 return -EINVAL;
4008 }
4009
4010 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4011 if (exec2_list == NULL) {
4012 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4013 args->buffer_count);
4014 return -ENOMEM;
4015 }
4016 ret = copy_from_user(exec2_list,
4017 (struct drm_i915_relocation_entry __user *)
4018 (uintptr_t) args->buffers_ptr,
4019 sizeof(*exec2_list) * args->buffer_count);
4020 if (ret != 0) {
4021 DRM_ERROR("copy %d exec entries failed %d\n",
4022 args->buffer_count, ret);
4023 drm_free_large(exec2_list);
4024 return -EFAULT;
4025 }
4026
4027 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4028 if (!ret) {
4029 /* Copy the new buffer offsets back to the user's exec list. */
4030 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4031 (uintptr_t) args->buffers_ptr,
4032 exec2_list,
4033 sizeof(*exec2_list) * args->buffer_count);
4034 if (ret) {
4035 ret = -EFAULT;
4036 DRM_ERROR("failed to copy %d exec entries "
4037 "back to user (%d)\n",
4038 args->buffer_count, ret);
4039 }
4040 }
4041
4042 drm_free_large(exec2_list);
4043 return ret;
4044}
4045
Eric Anholt673a3942008-07-30 12:06:12 -07004046int
4047i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4048{
4049 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01004050 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004051 int ret;
4052
Daniel Vetter778c3542010-05-13 11:49:44 +02004053 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4054
Eric Anholt673a3942008-07-30 12:06:12 -07004055 i915_verify_inactive(dev, __FILE__, __LINE__);
Chris Wilsonac0c6b52010-05-27 13:18:18 +01004056
4057 if (obj_priv->gtt_space != NULL) {
4058 if (alignment == 0)
4059 alignment = i915_gem_get_gtt_alignment(obj);
4060 if (obj_priv->gtt_offset & (alignment - 1)) {
Chris Wilsonae7d49d2010-08-04 12:37:41 +01004061 WARN(obj_priv->pin_count,
4062 "bo is already pinned with incorrect alignment:"
4063 " offset=%x, req.alignment=%x\n",
4064 obj_priv->gtt_offset, alignment);
Chris Wilsonac0c6b52010-05-27 13:18:18 +01004065 ret = i915_gem_object_unbind(obj);
4066 if (ret)
4067 return ret;
4068 }
4069 }
4070
Eric Anholt673a3942008-07-30 12:06:12 -07004071 if (obj_priv->gtt_space == NULL) {
4072 ret = i915_gem_object_bind_to_gtt(obj, alignment);
Chris Wilson97311292009-09-21 00:22:34 +01004073 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07004074 return ret;
Chris Wilson22c344e2009-02-11 14:26:45 +00004075 }
Jesse Barnes76446ca2009-12-17 22:05:42 -05004076
Eric Anholt673a3942008-07-30 12:06:12 -07004077 obj_priv->pin_count++;
4078
4079 /* If the object is not active and not pending a flush,
4080 * remove it from the inactive list
4081 */
4082 if (obj_priv->pin_count == 1) {
4083 atomic_inc(&dev->pin_count);
4084 atomic_add(obj->size, &dev->pin_memory);
4085 if (!obj_priv->active &&
Chris Wilsonbf1a1092010-08-07 11:01:20 +01004086 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
Eric Anholt673a3942008-07-30 12:06:12 -07004087 list_del_init(&obj_priv->list);
4088 }
4089 i915_verify_inactive(dev, __FILE__, __LINE__);
4090
4091 return 0;
4092}
4093
4094void
4095i915_gem_object_unpin(struct drm_gem_object *obj)
4096{
4097 struct drm_device *dev = obj->dev;
4098 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01004099 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004100
4101 i915_verify_inactive(dev, __FILE__, __LINE__);
4102 obj_priv->pin_count--;
4103 BUG_ON(obj_priv->pin_count < 0);
4104 BUG_ON(obj_priv->gtt_space == NULL);
4105
4106 /* If the object is no longer pinned, and is
4107 * neither active nor being flushed, then stick it on
4108 * the inactive list
4109 */
4110 if (obj_priv->pin_count == 0) {
4111 if (!obj_priv->active &&
Chris Wilson21d509e2009-06-06 09:46:02 +01004112 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
Eric Anholt673a3942008-07-30 12:06:12 -07004113 list_move_tail(&obj_priv->list,
4114 &dev_priv->mm.inactive_list);
4115 atomic_dec(&dev->pin_count);
4116 atomic_sub(obj->size, &dev->pin_memory);
4117 }
4118 i915_verify_inactive(dev, __FILE__, __LINE__);
4119}
4120
4121int
4122i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4123 struct drm_file *file_priv)
4124{
4125 struct drm_i915_gem_pin *args = data;
4126 struct drm_gem_object *obj;
4127 struct drm_i915_gem_object *obj_priv;
4128 int ret;
4129
4130 mutex_lock(&dev->struct_mutex);
4131
4132 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4133 if (obj == NULL) {
4134 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4135 args->handle);
4136 mutex_unlock(&dev->struct_mutex);
Chris Wilsonbf79cb92010-08-04 14:19:46 +01004137 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07004138 }
Daniel Vetter23010e42010-03-08 13:35:02 +01004139 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004140
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004141 if (obj_priv->madv != I915_MADV_WILLNEED) {
4142 DRM_ERROR("Attempting to pin a purgeable buffer\n");
Chris Wilson3ef94da2009-09-14 16:50:29 +01004143 drm_gem_object_unreference(obj);
4144 mutex_unlock(&dev->struct_mutex);
4145 return -EINVAL;
4146 }
4147
Jesse Barnes79e53942008-11-07 14:24:08 -08004148 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4149 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4150 args->handle);
Chris Wilson96dec612009-02-08 19:08:04 +00004151 drm_gem_object_unreference(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004152 mutex_unlock(&dev->struct_mutex);
Jesse Barnes79e53942008-11-07 14:24:08 -08004153 return -EINVAL;
4154 }
4155
4156 obj_priv->user_pin_count++;
4157 obj_priv->pin_filp = file_priv;
4158 if (obj_priv->user_pin_count == 1) {
4159 ret = i915_gem_object_pin(obj, args->alignment);
4160 if (ret != 0) {
4161 drm_gem_object_unreference(obj);
4162 mutex_unlock(&dev->struct_mutex);
4163 return ret;
4164 }
Eric Anholt673a3942008-07-30 12:06:12 -07004165 }
4166
4167 /* XXX - flush the CPU caches for pinned objects
4168 * as the X server doesn't manage domains yet
4169 */
Eric Anholte47c68e2008-11-14 13:35:19 -08004170 i915_gem_object_flush_cpu_write_domain(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004171 args->offset = obj_priv->gtt_offset;
4172 drm_gem_object_unreference(obj);
4173 mutex_unlock(&dev->struct_mutex);
4174
4175 return 0;
4176}
4177
4178int
4179i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4180 struct drm_file *file_priv)
4181{
4182 struct drm_i915_gem_pin *args = data;
4183 struct drm_gem_object *obj;
Jesse Barnes79e53942008-11-07 14:24:08 -08004184 struct drm_i915_gem_object *obj_priv;
Eric Anholt673a3942008-07-30 12:06:12 -07004185
4186 mutex_lock(&dev->struct_mutex);
4187
4188 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4189 if (obj == NULL) {
4190 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4191 args->handle);
4192 mutex_unlock(&dev->struct_mutex);
Chris Wilsonbf79cb92010-08-04 14:19:46 +01004193 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07004194 }
4195
Daniel Vetter23010e42010-03-08 13:35:02 +01004196 obj_priv = to_intel_bo(obj);
Jesse Barnes79e53942008-11-07 14:24:08 -08004197 if (obj_priv->pin_filp != file_priv) {
4198 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4199 args->handle);
4200 drm_gem_object_unreference(obj);
4201 mutex_unlock(&dev->struct_mutex);
4202 return -EINVAL;
4203 }
4204 obj_priv->user_pin_count--;
4205 if (obj_priv->user_pin_count == 0) {
4206 obj_priv->pin_filp = NULL;
4207 i915_gem_object_unpin(obj);
4208 }
Eric Anholt673a3942008-07-30 12:06:12 -07004209
4210 drm_gem_object_unreference(obj);
4211 mutex_unlock(&dev->struct_mutex);
4212 return 0;
4213}
4214
4215int
4216i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4217 struct drm_file *file_priv)
4218{
4219 struct drm_i915_gem_busy *args = data;
4220 struct drm_gem_object *obj;
4221 struct drm_i915_gem_object *obj_priv;
4222
Eric Anholt673a3942008-07-30 12:06:12 -07004223 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4224 if (obj == NULL) {
4225 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4226 args->handle);
Chris Wilsonbf79cb92010-08-04 14:19:46 +01004227 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07004228 }
4229
Chris Wilsonb1ce7862009-06-06 09:46:00 +01004230 mutex_lock(&dev->struct_mutex);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004231
Chris Wilson0be555b2010-08-04 15:36:30 +01004232 /* Count all active objects as busy, even if they are currently not used
4233 * by the gpu. Users of this interface expect objects to eventually
4234 * become non-busy without any further actions, therefore emit any
4235 * necessary flushes here.
Eric Anholtc4de0a52008-12-14 19:05:04 -08004236 */
Chris Wilson0be555b2010-08-04 15:36:30 +01004237 obj_priv = to_intel_bo(obj);
4238 args->busy = obj_priv->active;
4239 if (args->busy) {
4240 /* Unconditionally flush objects, even when the gpu still uses this
4241 * object. Userspace calling this function indicates that it wants to
4242 * use this buffer rather sooner than later, so issuing the required
4243 * flush earlier is beneficial.
4244 */
4245 if (obj->write_domain) {
4246 i915_gem_flush(dev, 0, obj->write_domain);
4247 (void)i915_add_request(dev, file_priv, obj->write_domain, obj_priv->ring);
4248 }
4249
4250 /* Update the active list for the hardware's current position.
4251 * Otherwise this only updates on a delayed timer or when irqs
4252 * are actually unmasked, and our working set ends up being
4253 * larger than required.
4254 */
4255 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4256
4257 args->busy = obj_priv->active;
4258 }
Eric Anholt673a3942008-07-30 12:06:12 -07004259
4260 drm_gem_object_unreference(obj);
4261 mutex_unlock(&dev->struct_mutex);
4262 return 0;
4263}
4264
4265int
4266i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4267 struct drm_file *file_priv)
4268{
4269 return i915_gem_ring_throttle(dev, file_priv);
4270}
4271
Chris Wilson3ef94da2009-09-14 16:50:29 +01004272int
4273i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4274 struct drm_file *file_priv)
4275{
4276 struct drm_i915_gem_madvise *args = data;
4277 struct drm_gem_object *obj;
4278 struct drm_i915_gem_object *obj_priv;
4279
4280 switch (args->madv) {
4281 case I915_MADV_DONTNEED:
4282 case I915_MADV_WILLNEED:
4283 break;
4284 default:
4285 return -EINVAL;
4286 }
4287
4288 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4289 if (obj == NULL) {
4290 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4291 args->handle);
Chris Wilsonbf79cb92010-08-04 14:19:46 +01004292 return -ENOENT;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004293 }
4294
4295 mutex_lock(&dev->struct_mutex);
Daniel Vetter23010e42010-03-08 13:35:02 +01004296 obj_priv = to_intel_bo(obj);
Chris Wilson3ef94da2009-09-14 16:50:29 +01004297
4298 if (obj_priv->pin_count) {
4299 drm_gem_object_unreference(obj);
4300 mutex_unlock(&dev->struct_mutex);
4301
4302 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4303 return -EINVAL;
4304 }
4305
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004306 if (obj_priv->madv != __I915_MADV_PURGED)
4307 obj_priv->madv = args->madv;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004308
Chris Wilson2d7ef392009-09-20 23:13:10 +01004309 /* if the object is no longer bound, discard its backing storage */
4310 if (i915_gem_object_is_purgeable(obj_priv) &&
4311 obj_priv->gtt_space == NULL)
4312 i915_gem_object_truncate(obj);
4313
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004314 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4315
Chris Wilson3ef94da2009-09-14 16:50:29 +01004316 drm_gem_object_unreference(obj);
4317 mutex_unlock(&dev->struct_mutex);
4318
4319 return 0;
4320}
4321
Daniel Vetterac52bc52010-04-09 19:05:06 +00004322struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4323 size_t size)
4324{
Daniel Vetterc397b902010-04-09 19:05:07 +00004325 struct drm_i915_gem_object *obj;
4326
4327 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4328 if (obj == NULL)
4329 return NULL;
4330
4331 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4332 kfree(obj);
4333 return NULL;
4334 }
4335
4336 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4337 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4338
4339 obj->agp_type = AGP_USER_MEMORY;
Daniel Vetter62b8b212010-04-09 19:05:08 +00004340 obj->base.driver_private = NULL;
Daniel Vetterc397b902010-04-09 19:05:07 +00004341 obj->fence_reg = I915_FENCE_REG_NONE;
4342 INIT_LIST_HEAD(&obj->list);
4343 INIT_LIST_HEAD(&obj->gpu_write_list);
Daniel Vetterc397b902010-04-09 19:05:07 +00004344 obj->madv = I915_MADV_WILLNEED;
4345
4346 trace_i915_gem_object_create(&obj->base);
4347
4348 return &obj->base;
Daniel Vetterac52bc52010-04-09 19:05:06 +00004349}
4350
Eric Anholt673a3942008-07-30 12:06:12 -07004351int i915_gem_init_object(struct drm_gem_object *obj)
4352{
Daniel Vetterc397b902010-04-09 19:05:07 +00004353 BUG();
Jesse Barnesde151cf2008-11-12 10:03:55 -08004354
Eric Anholt673a3942008-07-30 12:06:12 -07004355 return 0;
4356}
4357
Chris Wilsonbe726152010-07-23 23:18:50 +01004358static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4359{
4360 struct drm_device *dev = obj->dev;
4361 drm_i915_private_t *dev_priv = dev->dev_private;
4362 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4363 int ret;
4364
4365 ret = i915_gem_object_unbind(obj);
4366 if (ret == -ERESTARTSYS) {
4367 list_move(&obj_priv->list,
4368 &dev_priv->mm.deferred_free_list);
4369 return;
4370 }
4371
4372 if (obj_priv->mmap_offset)
4373 i915_gem_free_mmap_offset(obj);
4374
4375 drm_gem_object_release(obj);
4376
4377 kfree(obj_priv->page_cpu_valid);
4378 kfree(obj_priv->bit_17);
4379 kfree(obj_priv);
4380}
4381
Eric Anholt673a3942008-07-30 12:06:12 -07004382void i915_gem_free_object(struct drm_gem_object *obj)
4383{
Jesse Barnesde151cf2008-11-12 10:03:55 -08004384 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01004385 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004386
Chris Wilson1c5d22f2009-08-25 11:15:50 +01004387 trace_i915_gem_object_destroy(obj);
4388
Eric Anholt673a3942008-07-30 12:06:12 -07004389 while (obj_priv->pin_count > 0)
4390 i915_gem_object_unpin(obj);
4391
Dave Airlie71acb5e2008-12-30 20:31:46 +10004392 if (obj_priv->phys_obj)
4393 i915_gem_detach_phys_object(dev, obj);
4394
Chris Wilsonbe726152010-07-23 23:18:50 +01004395 i915_gem_free_object_tail(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004396}
4397
Jesse Barnes5669fca2009-02-17 15:13:31 -08004398int
Eric Anholt673a3942008-07-30 12:06:12 -07004399i915_gem_idle(struct drm_device *dev)
4400{
4401 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson29105cc2010-01-07 10:39:13 +00004402 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004403
Keith Packard6dbe2772008-10-14 21:41:13 -07004404 mutex_lock(&dev->struct_mutex);
4405
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004406 if (dev_priv->mm.suspended ||
Zou Nan haid1b851f2010-05-21 09:08:57 +08004407 (dev_priv->render_ring.gem_object == NULL) ||
4408 (HAS_BSD(dev) &&
4409 dev_priv->bsd_ring.gem_object == NULL)) {
Keith Packard6dbe2772008-10-14 21:41:13 -07004410 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07004411 return 0;
Keith Packard6dbe2772008-10-14 21:41:13 -07004412 }
Eric Anholt673a3942008-07-30 12:06:12 -07004413
Chris Wilson29105cc2010-01-07 10:39:13 +00004414 ret = i915_gpu_idle(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07004415 if (ret) {
4416 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07004417 return ret;
Keith Packard6dbe2772008-10-14 21:41:13 -07004418 }
Eric Anholt673a3942008-07-30 12:06:12 -07004419
Chris Wilson29105cc2010-01-07 10:39:13 +00004420 /* Under UMS, be paranoid and evict. */
4421 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
Chris Wilsonb47eb4a2010-08-07 11:01:23 +01004422 ret = i915_gem_evict_inactive(dev);
Chris Wilson29105cc2010-01-07 10:39:13 +00004423 if (ret) {
4424 mutex_unlock(&dev->struct_mutex);
4425 return ret;
4426 }
4427 }
4428
4429 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4430 * We need to replace this with a semaphore, or something.
4431 * And not confound mm.suspended!
4432 */
4433 dev_priv->mm.suspended = 1;
Daniel Vetterbc0c7f12010-08-20 18:18:48 +02004434 del_timer_sync(&dev_priv->hangcheck_timer);
Chris Wilson29105cc2010-01-07 10:39:13 +00004435
4436 i915_kernel_lost_context(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07004437 i915_gem_cleanup_ringbuffer(dev);
Chris Wilson29105cc2010-01-07 10:39:13 +00004438
Keith Packard6dbe2772008-10-14 21:41:13 -07004439 mutex_unlock(&dev->struct_mutex);
4440
Chris Wilson29105cc2010-01-07 10:39:13 +00004441 /* Cancel the retire work handler, which should be idle now. */
4442 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4443
Eric Anholt673a3942008-07-30 12:06:12 -07004444 return 0;
4445}
4446
Jesse Barnese552eb72010-04-21 11:39:23 -07004447/*
4448 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4449 * over cache flushing.
4450 */
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004451static int
Jesse Barnese552eb72010-04-21 11:39:23 -07004452i915_gem_init_pipe_control(struct drm_device *dev)
4453{
4454 drm_i915_private_t *dev_priv = dev->dev_private;
4455 struct drm_gem_object *obj;
4456 struct drm_i915_gem_object *obj_priv;
4457 int ret;
4458
Eric Anholt34dc4d42010-05-07 14:30:03 -07004459 obj = i915_gem_alloc_object(dev, 4096);
Jesse Barnese552eb72010-04-21 11:39:23 -07004460 if (obj == NULL) {
4461 DRM_ERROR("Failed to allocate seqno page\n");
4462 ret = -ENOMEM;
4463 goto err;
4464 }
4465 obj_priv = to_intel_bo(obj);
4466 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4467
4468 ret = i915_gem_object_pin(obj, 4096);
4469 if (ret)
4470 goto err_unref;
4471
4472 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4473 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4474 if (dev_priv->seqno_page == NULL)
4475 goto err_unpin;
4476
4477 dev_priv->seqno_obj = obj;
4478 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4479
4480 return 0;
4481
4482err_unpin:
4483 i915_gem_object_unpin(obj);
4484err_unref:
4485 drm_gem_object_unreference(obj);
4486err:
4487 return ret;
4488}
4489
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004490
4491static void
Jesse Barnese552eb72010-04-21 11:39:23 -07004492i915_gem_cleanup_pipe_control(struct drm_device *dev)
4493{
4494 drm_i915_private_t *dev_priv = dev->dev_private;
4495 struct drm_gem_object *obj;
4496 struct drm_i915_gem_object *obj_priv;
4497
4498 obj = dev_priv->seqno_obj;
4499 obj_priv = to_intel_bo(obj);
4500 kunmap(obj_priv->pages[0]);
4501 i915_gem_object_unpin(obj);
4502 drm_gem_object_unreference(obj);
4503 dev_priv->seqno_obj = NULL;
4504
4505 dev_priv->seqno_page = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07004506}
4507
Eric Anholt673a3942008-07-30 12:06:12 -07004508int
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004509i915_gem_init_ringbuffer(struct drm_device *dev)
4510{
4511 drm_i915_private_t *dev_priv = dev->dev_private;
4512 int ret;
Chris Wilson68f95ba2010-05-27 13:18:22 +01004513
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004514 dev_priv->render_ring = render_ring;
Chris Wilson68f95ba2010-05-27 13:18:22 +01004515
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004516 if (!I915_NEED_GFX_HWS(dev)) {
4517 dev_priv->render_ring.status_page.page_addr
4518 = dev_priv->status_page_dmah->vaddr;
4519 memset(dev_priv->render_ring.status_page.page_addr,
4520 0, PAGE_SIZE);
4521 }
Chris Wilson68f95ba2010-05-27 13:18:22 +01004522
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004523 if (HAS_PIPE_CONTROL(dev)) {
4524 ret = i915_gem_init_pipe_control(dev);
4525 if (ret)
4526 return ret;
4527 }
Chris Wilson68f95ba2010-05-27 13:18:22 +01004528
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004529 ret = intel_init_ring_buffer(dev, &dev_priv->render_ring);
Chris Wilson68f95ba2010-05-27 13:18:22 +01004530 if (ret)
4531 goto cleanup_pipe_control;
4532
4533 if (HAS_BSD(dev)) {
Zou Nan haid1b851f2010-05-21 09:08:57 +08004534 dev_priv->bsd_ring = bsd_ring;
4535 ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
Chris Wilson68f95ba2010-05-27 13:18:22 +01004536 if (ret)
4537 goto cleanup_render_ring;
Zou Nan haid1b851f2010-05-21 09:08:57 +08004538 }
Chris Wilson68f95ba2010-05-27 13:18:22 +01004539
Chris Wilson6f392d52010-08-07 11:01:22 +01004540 dev_priv->next_seqno = 1;
4541
Chris Wilson68f95ba2010-05-27 13:18:22 +01004542 return 0;
4543
4544cleanup_render_ring:
4545 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4546cleanup_pipe_control:
4547 if (HAS_PIPE_CONTROL(dev))
4548 i915_gem_cleanup_pipe_control(dev);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004549 return ret;
4550}
4551
4552void
4553i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4554{
4555 drm_i915_private_t *dev_priv = dev->dev_private;
4556
4557 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004558 if (HAS_BSD(dev))
4559 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004560 if (HAS_PIPE_CONTROL(dev))
4561 i915_gem_cleanup_pipe_control(dev);
4562}
4563
4564int
Eric Anholt673a3942008-07-30 12:06:12 -07004565i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4566 struct drm_file *file_priv)
4567{
4568 drm_i915_private_t *dev_priv = dev->dev_private;
4569 int ret;
4570
Jesse Barnes79e53942008-11-07 14:24:08 -08004571 if (drm_core_check_feature(dev, DRIVER_MODESET))
4572 return 0;
4573
Ben Gamariba1234d2009-09-14 17:48:47 -04004574 if (atomic_read(&dev_priv->mm.wedged)) {
Eric Anholt673a3942008-07-30 12:06:12 -07004575 DRM_ERROR("Reenabling wedged hardware, good luck\n");
Ben Gamariba1234d2009-09-14 17:48:47 -04004576 atomic_set(&dev_priv->mm.wedged, 0);
Eric Anholt673a3942008-07-30 12:06:12 -07004577 }
4578
Eric Anholt673a3942008-07-30 12:06:12 -07004579 mutex_lock(&dev->struct_mutex);
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004580 dev_priv->mm.suspended = 0;
4581
4582 ret = i915_gem_init_ringbuffer(dev);
Wu Fengguangd816f6a2009-04-18 10:43:32 +08004583 if (ret != 0) {
4584 mutex_unlock(&dev->struct_mutex);
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004585 return ret;
Wu Fengguangd816f6a2009-04-18 10:43:32 +08004586 }
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004587
Carl Worth5e118f42009-03-20 11:54:25 -07004588 spin_lock(&dev_priv->mm.active_list_lock);
Zou Nan hai852835f2010-05-21 09:08:56 +08004589 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
Zou Nan haid1b851f2010-05-21 09:08:57 +08004590 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
Carl Worth5e118f42009-03-20 11:54:25 -07004591 spin_unlock(&dev_priv->mm.active_list_lock);
4592
Eric Anholt673a3942008-07-30 12:06:12 -07004593 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4594 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
Zou Nan hai852835f2010-05-21 09:08:56 +08004595 BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
Zou Nan haid1b851f2010-05-21 09:08:57 +08004596 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
Eric Anholt673a3942008-07-30 12:06:12 -07004597 mutex_unlock(&dev->struct_mutex);
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004598
Chris Wilson5f353082010-06-07 14:03:03 +01004599 ret = drm_irq_install(dev);
4600 if (ret)
4601 goto cleanup_ringbuffer;
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004602
Eric Anholt673a3942008-07-30 12:06:12 -07004603 return 0;
Chris Wilson5f353082010-06-07 14:03:03 +01004604
4605cleanup_ringbuffer:
4606 mutex_lock(&dev->struct_mutex);
4607 i915_gem_cleanup_ringbuffer(dev);
4608 dev_priv->mm.suspended = 1;
4609 mutex_unlock(&dev->struct_mutex);
4610
4611 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004612}
4613
4614int
4615i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4616 struct drm_file *file_priv)
4617{
Jesse Barnes79e53942008-11-07 14:24:08 -08004618 if (drm_core_check_feature(dev, DRIVER_MODESET))
4619 return 0;
4620
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004621 drm_irq_uninstall(dev);
Linus Torvaldse6890f62009-09-08 17:09:24 -07004622 return i915_gem_idle(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07004623}
4624
4625void
4626i915_gem_lastclose(struct drm_device *dev)
4627{
4628 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004629
Eric Anholte806b492009-01-22 09:56:58 -08004630 if (drm_core_check_feature(dev, DRIVER_MODESET))
4631 return;
4632
Keith Packard6dbe2772008-10-14 21:41:13 -07004633 ret = i915_gem_idle(dev);
4634 if (ret)
4635 DRM_ERROR("failed to idle hardware: %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07004636}
4637
4638void
4639i915_gem_load(struct drm_device *dev)
4640{
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004641 int i;
Eric Anholt673a3942008-07-30 12:06:12 -07004642 drm_i915_private_t *dev_priv = dev->dev_private;
4643
Carl Worth5e118f42009-03-20 11:54:25 -07004644 spin_lock_init(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07004645 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
Daniel Vetter99fcb762010-02-07 16:20:18 +01004646 INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004647 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07004648 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
Chris Wilsonbe726152010-07-23 23:18:50 +01004649 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
Zou Nan hai852835f2010-05-21 09:08:56 +08004650 INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4651 INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004652 if (HAS_BSD(dev)) {
4653 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4654 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4655 }
Daniel Vetter007cc8a2010-04-28 11:02:31 +02004656 for (i = 0; i < 16; i++)
4657 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004658 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4659 i915_gem_retire_work_handler);
Chris Wilson31169712009-09-14 16:50:28 +01004660 spin_lock(&shrink_list_lock);
4661 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4662 spin_unlock(&shrink_list_lock);
4663
Dave Airlie94400122010-07-20 13:15:31 +10004664 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4665 if (IS_GEN3(dev)) {
4666 u32 tmp = I915_READ(MI_ARB_STATE);
4667 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4668 /* arb state is a masked write, so set bit + bit in mask */
4669 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4670 I915_WRITE(MI_ARB_STATE, tmp);
4671 }
4672 }
4673
Jesse Barnesde151cf2008-11-12 10:03:55 -08004674 /* Old X drivers will take 0-2 for front, back, depth buffers */
Eric Anholtb397c832010-01-26 09:43:10 -08004675 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4676 dev_priv->fence_reg_start = 3;
Jesse Barnesde151cf2008-11-12 10:03:55 -08004677
Jesse Barnes0f973f22009-01-26 17:10:45 -08004678 if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
Jesse Barnesde151cf2008-11-12 10:03:55 -08004679 dev_priv->num_fence_regs = 16;
4680 else
4681 dev_priv->num_fence_regs = 8;
4682
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004683 /* Initialize fence registers to zero */
4684 if (IS_I965G(dev)) {
4685 for (i = 0; i < 16; i++)
4686 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4687 } else {
4688 for (i = 0; i < 8; i++)
4689 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4690 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4691 for (i = 0; i < 8; i++)
4692 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4693 }
Eric Anholt673a3942008-07-30 12:06:12 -07004694 i915_gem_detect_bit_6_swizzle(dev);
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05004695 init_waitqueue_head(&dev_priv->pending_flip_queue);
Eric Anholt673a3942008-07-30 12:06:12 -07004696}
Dave Airlie71acb5e2008-12-30 20:31:46 +10004697
4698/*
4699 * Create a physically contiguous memory object for this object
4700 * e.g. for cursor + overlay regs
4701 */
4702int i915_gem_init_phys_object(struct drm_device *dev,
Chris Wilson6eeefaf2010-08-07 11:01:39 +01004703 int id, int size, int align)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004704{
4705 drm_i915_private_t *dev_priv = dev->dev_private;
4706 struct drm_i915_gem_phys_object *phys_obj;
4707 int ret;
4708
4709 if (dev_priv->mm.phys_objs[id - 1] || !size)
4710 return 0;
4711
Eric Anholt9a298b22009-03-24 12:23:04 -07004712 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004713 if (!phys_obj)
4714 return -ENOMEM;
4715
4716 phys_obj->id = id;
4717
Chris Wilson6eeefaf2010-08-07 11:01:39 +01004718 phys_obj->handle = drm_pci_alloc(dev, size, align);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004719 if (!phys_obj->handle) {
4720 ret = -ENOMEM;
4721 goto kfree_obj;
4722 }
4723#ifdef CONFIG_X86
4724 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4725#endif
4726
4727 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4728
4729 return 0;
4730kfree_obj:
Eric Anholt9a298b22009-03-24 12:23:04 -07004731 kfree(phys_obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004732 return ret;
4733}
4734
4735void i915_gem_free_phys_object(struct drm_device *dev, int id)
4736{
4737 drm_i915_private_t *dev_priv = dev->dev_private;
4738 struct drm_i915_gem_phys_object *phys_obj;
4739
4740 if (!dev_priv->mm.phys_objs[id - 1])
4741 return;
4742
4743 phys_obj = dev_priv->mm.phys_objs[id - 1];
4744 if (phys_obj->cur_obj) {
4745 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4746 }
4747
4748#ifdef CONFIG_X86
4749 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4750#endif
4751 drm_pci_free(dev, phys_obj->handle);
4752 kfree(phys_obj);
4753 dev_priv->mm.phys_objs[id - 1] = NULL;
4754}
4755
4756void i915_gem_free_all_phys_object(struct drm_device *dev)
4757{
4758 int i;
4759
Dave Airlie260883c2009-01-22 17:58:49 +10004760 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004761 i915_gem_free_phys_object(dev, i);
4762}
4763
4764void i915_gem_detach_phys_object(struct drm_device *dev,
4765 struct drm_gem_object *obj)
4766{
4767 struct drm_i915_gem_object *obj_priv;
4768 int i;
4769 int ret;
4770 int page_count;
4771
Daniel Vetter23010e42010-03-08 13:35:02 +01004772 obj_priv = to_intel_bo(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004773 if (!obj_priv->phys_obj)
4774 return;
4775
Chris Wilson4bdadb92010-01-27 13:36:32 +00004776 ret = i915_gem_object_get_pages(obj, 0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004777 if (ret)
4778 goto out;
4779
4780 page_count = obj->size / PAGE_SIZE;
4781
4782 for (i = 0; i < page_count; i++) {
Eric Anholt856fa192009-03-19 14:10:50 -07004783 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004784 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4785
4786 memcpy(dst, src, PAGE_SIZE);
4787 kunmap_atomic(dst, KM_USER0);
4788 }
Eric Anholt856fa192009-03-19 14:10:50 -07004789 drm_clflush_pages(obj_priv->pages, page_count);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004790 drm_agp_chipset_flush(dev);
Chris Wilsond78b47b2009-06-17 21:52:49 +01004791
4792 i915_gem_object_put_pages(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004793out:
4794 obj_priv->phys_obj->cur_obj = NULL;
4795 obj_priv->phys_obj = NULL;
4796}
4797
4798int
4799i915_gem_attach_phys_object(struct drm_device *dev,
Chris Wilson6eeefaf2010-08-07 11:01:39 +01004800 struct drm_gem_object *obj,
4801 int id,
4802 int align)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004803{
4804 drm_i915_private_t *dev_priv = dev->dev_private;
4805 struct drm_i915_gem_object *obj_priv;
4806 int ret = 0;
4807 int page_count;
4808 int i;
4809
4810 if (id > I915_MAX_PHYS_OBJECT)
4811 return -EINVAL;
4812
Daniel Vetter23010e42010-03-08 13:35:02 +01004813 obj_priv = to_intel_bo(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004814
4815 if (obj_priv->phys_obj) {
4816 if (obj_priv->phys_obj->id == id)
4817 return 0;
4818 i915_gem_detach_phys_object(dev, obj);
4819 }
4820
Dave Airlie71acb5e2008-12-30 20:31:46 +10004821 /* create a new object */
4822 if (!dev_priv->mm.phys_objs[id - 1]) {
4823 ret = i915_gem_init_phys_object(dev, id,
Chris Wilson6eeefaf2010-08-07 11:01:39 +01004824 obj->size, align);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004825 if (ret) {
Linus Torvaldsaeb565d2009-01-26 10:01:53 -08004826 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004827 goto out;
4828 }
4829 }
4830
4831 /* bind to the object */
4832 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4833 obj_priv->phys_obj->cur_obj = obj;
4834
Chris Wilson4bdadb92010-01-27 13:36:32 +00004835 ret = i915_gem_object_get_pages(obj, 0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004836 if (ret) {
4837 DRM_ERROR("failed to get page list\n");
4838 goto out;
4839 }
4840
4841 page_count = obj->size / PAGE_SIZE;
4842
4843 for (i = 0; i < page_count; i++) {
Eric Anholt856fa192009-03-19 14:10:50 -07004844 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004845 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4846
4847 memcpy(dst, src, PAGE_SIZE);
4848 kunmap_atomic(src, KM_USER0);
4849 }
4850
Chris Wilsond78b47b2009-06-17 21:52:49 +01004851 i915_gem_object_put_pages(obj);
4852
Dave Airlie71acb5e2008-12-30 20:31:46 +10004853 return 0;
4854out:
4855 return ret;
4856}
4857
4858static int
4859i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4860 struct drm_i915_gem_pwrite *args,
4861 struct drm_file *file_priv)
4862{
Daniel Vetter23010e42010-03-08 13:35:02 +01004863 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004864 void *obj_addr;
4865 int ret;
4866 char __user *user_data;
4867
4868 user_data = (char __user *) (uintptr_t) args->data_ptr;
4869 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4870
Zhao Yakui44d98a62009-10-09 11:39:40 +08004871 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004872 ret = copy_from_user(obj_addr, user_data, args->size);
4873 if (ret)
4874 return -EFAULT;
4875
4876 drm_agp_chipset_flush(dev);
4877 return 0;
4878}
Eric Anholtb9624422009-06-03 07:27:35 +00004879
4880void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4881{
4882 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4883
4884 /* Clean up our request list when the client is going away, so that
4885 * later retire_requests won't dereference our soon-to-be-gone
4886 * file_priv.
4887 */
4888 mutex_lock(&dev->struct_mutex);
4889 while (!list_empty(&i915_file_priv->mm.request_list))
4890 list_del_init(i915_file_priv->mm.request_list.next);
4891 mutex_unlock(&dev->struct_mutex);
4892}
Chris Wilson31169712009-09-14 16:50:28 +01004893
Chris Wilson31169712009-09-14 16:50:28 +01004894static int
Chris Wilson1637ef42010-04-20 17:10:35 +01004895i915_gpu_is_active(struct drm_device *dev)
4896{
4897 drm_i915_private_t *dev_priv = dev->dev_private;
4898 int lists_empty;
4899
4900 spin_lock(&dev_priv->mm.active_list_lock);
4901 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
Zou Nan hai852835f2010-05-21 09:08:56 +08004902 list_empty(&dev_priv->render_ring.active_list);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004903 if (HAS_BSD(dev))
4904 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
Chris Wilson1637ef42010-04-20 17:10:35 +01004905 spin_unlock(&dev_priv->mm.active_list_lock);
4906
4907 return !lists_empty;
4908}
4909
4910static int
Dave Chinner7f8275d2010-07-19 14:56:17 +10004911i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
Chris Wilson31169712009-09-14 16:50:28 +01004912{
4913 drm_i915_private_t *dev_priv, *next_dev;
4914 struct drm_i915_gem_object *obj_priv, *next_obj;
4915 int cnt = 0;
4916 int would_deadlock = 1;
4917
4918 /* "fast-path" to count number of available objects */
4919 if (nr_to_scan == 0) {
4920 spin_lock(&shrink_list_lock);
4921 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4922 struct drm_device *dev = dev_priv->dev;
4923
4924 if (mutex_trylock(&dev->struct_mutex)) {
4925 list_for_each_entry(obj_priv,
4926 &dev_priv->mm.inactive_list,
4927 list)
4928 cnt++;
4929 mutex_unlock(&dev->struct_mutex);
4930 }
4931 }
4932 spin_unlock(&shrink_list_lock);
4933
4934 return (cnt / 100) * sysctl_vfs_cache_pressure;
4935 }
4936
4937 spin_lock(&shrink_list_lock);
4938
Chris Wilson1637ef42010-04-20 17:10:35 +01004939rescan:
Chris Wilson31169712009-09-14 16:50:28 +01004940 /* first scan for clean buffers */
4941 list_for_each_entry_safe(dev_priv, next_dev,
4942 &shrink_list, mm.shrink_list) {
4943 struct drm_device *dev = dev_priv->dev;
4944
4945 if (! mutex_trylock(&dev->struct_mutex))
4946 continue;
4947
4948 spin_unlock(&shrink_list_lock);
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01004949 i915_gem_retire_requests(dev);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004950
Chris Wilson31169712009-09-14 16:50:28 +01004951 list_for_each_entry_safe(obj_priv, next_obj,
4952 &dev_priv->mm.inactive_list,
4953 list) {
4954 if (i915_gem_object_is_purgeable(obj_priv)) {
Daniel Vettera8089e82010-04-09 19:05:09 +00004955 i915_gem_object_unbind(&obj_priv->base);
Chris Wilson31169712009-09-14 16:50:28 +01004956 if (--nr_to_scan <= 0)
4957 break;
4958 }
4959 }
4960
4961 spin_lock(&shrink_list_lock);
4962 mutex_unlock(&dev->struct_mutex);
4963
Chris Wilson963b4832009-09-20 23:03:54 +01004964 would_deadlock = 0;
4965
Chris Wilson31169712009-09-14 16:50:28 +01004966 if (nr_to_scan <= 0)
4967 break;
4968 }
4969
4970 /* second pass, evict/count anything still on the inactive list */
4971 list_for_each_entry_safe(dev_priv, next_dev,
4972 &shrink_list, mm.shrink_list) {
4973 struct drm_device *dev = dev_priv->dev;
4974
4975 if (! mutex_trylock(&dev->struct_mutex))
4976 continue;
4977
4978 spin_unlock(&shrink_list_lock);
4979
4980 list_for_each_entry_safe(obj_priv, next_obj,
4981 &dev_priv->mm.inactive_list,
4982 list) {
4983 if (nr_to_scan > 0) {
Daniel Vettera8089e82010-04-09 19:05:09 +00004984 i915_gem_object_unbind(&obj_priv->base);
Chris Wilson31169712009-09-14 16:50:28 +01004985 nr_to_scan--;
4986 } else
4987 cnt++;
4988 }
4989
4990 spin_lock(&shrink_list_lock);
4991 mutex_unlock(&dev->struct_mutex);
4992
4993 would_deadlock = 0;
4994 }
4995
Chris Wilson1637ef42010-04-20 17:10:35 +01004996 if (nr_to_scan) {
4997 int active = 0;
4998
4999 /*
5000 * We are desperate for pages, so as a last resort, wait
5001 * for the GPU to finish and discard whatever we can.
5002 * This has a dramatic impact to reduce the number of
5003 * OOM-killer events whilst running the GPU aggressively.
5004 */
5005 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5006 struct drm_device *dev = dev_priv->dev;
5007
5008 if (!mutex_trylock(&dev->struct_mutex))
5009 continue;
5010
5011 spin_unlock(&shrink_list_lock);
5012
5013 if (i915_gpu_is_active(dev)) {
5014 i915_gpu_idle(dev);
5015 active++;
5016 }
5017
5018 spin_lock(&shrink_list_lock);
5019 mutex_unlock(&dev->struct_mutex);
5020 }
5021
5022 if (active)
5023 goto rescan;
5024 }
5025
Chris Wilson31169712009-09-14 16:50:28 +01005026 spin_unlock(&shrink_list_lock);
5027
5028 if (would_deadlock)
5029 return -1;
5030 else if (cnt > 0)
5031 return (cnt / 100) * sysctl_vfs_cache_pressure;
5032 else
5033 return 0;
5034}
5035
5036static struct shrinker shrinker = {
5037 .shrink = i915_gem_shrink,
5038 .seeks = DEFAULT_SEEKS,
5039};
5040
5041__init void
5042i915_gem_shrinker_init(void)
5043{
5044 register_shrinker(&shrinker);
5045}
5046
5047__exit void
5048i915_gem_shrinker_exit(void)
5049{
5050 unregister_shrinker(&shrinker);
5051}