blob: 715eaac62dbdb95683caef4b2516a9da043de9f0 [file] [log] [blame]
Eric Anholt673a3942008-07-30 12:06:12 -07001/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
Chris Wilson1c5d22f2009-08-25 11:15:50 +010032#include "i915_trace.h"
Jesse Barnes652c3932009-08-17 13:31:43 -070033#include "intel_drv.h"
Eric Anholt673a3942008-07-30 12:06:12 -070034#include <linux/swap.h>
Jesse Barnes79e53942008-11-07 14:24:08 -080035#include <linux/pci.h>
Eric Anholt673a3942008-07-30 12:06:12 -070036
Eric Anholt28dfe522008-11-13 15:00:55 -080037#define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
38
Eric Anholte47c68e2008-11-14 13:35:19 -080039static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
40static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
41static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
Eric Anholte47c68e2008-11-14 13:35:19 -080042static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
43 int write);
44static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
45 uint64_t offset,
46 uint64_t size);
47static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
Eric Anholt673a3942008-07-30 12:06:12 -070048static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -080049static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
50 unsigned alignment);
Jesse Barnesde151cf2008-11-12 10:03:55 -080051static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
Chris Wilson07f73f62009-09-14 16:50:30 +010052static int i915_gem_evict_something(struct drm_device *dev, int min_size);
Chris Wilsonab5ee572009-09-20 19:25:47 +010053static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
Dave Airlie71acb5e2008-12-30 20:31:46 +100054static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
55 struct drm_i915_gem_pwrite *args,
56 struct drm_file *file_priv);
Eric Anholt673a3942008-07-30 12:06:12 -070057
Chris Wilson31169712009-09-14 16:50:28 +010058static LIST_HEAD(shrink_list);
59static DEFINE_SPINLOCK(shrink_list_lock);
60
Jesse Barnes79e53942008-11-07 14:24:08 -080061int i915_gem_do_init(struct drm_device *dev, unsigned long start,
62 unsigned long end)
63{
64 drm_i915_private_t *dev_priv = dev->dev_private;
65
66 if (start >= end ||
67 (start & (PAGE_SIZE - 1)) != 0 ||
68 (end & (PAGE_SIZE - 1)) != 0) {
69 return -EINVAL;
70 }
71
72 drm_mm_init(&dev_priv->mm.gtt_space, start,
73 end - start);
74
75 dev->gtt_total = (uint32_t) (end - start);
76
77 return 0;
78}
Keith Packard6dbe2772008-10-14 21:41:13 -070079
Eric Anholt673a3942008-07-30 12:06:12 -070080int
81i915_gem_init_ioctl(struct drm_device *dev, void *data,
82 struct drm_file *file_priv)
83{
Eric Anholt673a3942008-07-30 12:06:12 -070084 struct drm_i915_gem_init *args = data;
Jesse Barnes79e53942008-11-07 14:24:08 -080085 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -070086
87 mutex_lock(&dev->struct_mutex);
Jesse Barnes79e53942008-11-07 14:24:08 -080088 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
Eric Anholt673a3942008-07-30 12:06:12 -070089 mutex_unlock(&dev->struct_mutex);
90
Jesse Barnes79e53942008-11-07 14:24:08 -080091 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -070092}
93
Eric Anholt5a125c32008-10-22 21:40:13 -070094int
95i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
96 struct drm_file *file_priv)
97{
Eric Anholt5a125c32008-10-22 21:40:13 -070098 struct drm_i915_gem_get_aperture *args = data;
Eric Anholt5a125c32008-10-22 21:40:13 -070099
100 if (!(dev->driver->driver_features & DRIVER_GEM))
101 return -ENODEV;
102
103 args->aper_size = dev->gtt_total;
Keith Packard2678d9d2008-11-20 22:54:54 -0800104 args->aper_available_size = (args->aper_size -
105 atomic_read(&dev->pin_memory));
Eric Anholt5a125c32008-10-22 21:40:13 -0700106
107 return 0;
108}
109
Eric Anholt673a3942008-07-30 12:06:12 -0700110
111/**
112 * Creates a new mm object and returns a handle to it.
113 */
114int
115i915_gem_create_ioctl(struct drm_device *dev, void *data,
116 struct drm_file *file_priv)
117{
118 struct drm_i915_gem_create *args = data;
119 struct drm_gem_object *obj;
Pekka Paalanena1a2d1d2009-08-23 12:40:55 +0300120 int ret;
121 u32 handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700122
123 args->size = roundup(args->size, PAGE_SIZE);
124
125 /* Allocate the new object */
126 obj = drm_gem_object_alloc(dev, args->size);
127 if (obj == NULL)
128 return -ENOMEM;
129
130 ret = drm_gem_handle_create(file_priv, obj, &handle);
Luca Barbieribc9025b2010-02-09 05:49:12 +0000131 drm_gem_object_handle_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700132
133 if (ret)
134 return ret;
135
136 args->handle = handle;
137
138 return 0;
139}
140
Eric Anholt40123c12009-03-09 13:42:30 -0700141static inline int
Eric Anholteb014592009-03-10 11:44:52 -0700142fast_shmem_read(struct page **pages,
143 loff_t page_base, int page_offset,
144 char __user *data,
145 int length)
146{
147 char __iomem *vaddr;
Florian Mickler2bc43b52009-04-06 22:55:41 +0200148 int unwritten;
Eric Anholteb014592009-03-10 11:44:52 -0700149
150 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
151 if (vaddr == NULL)
152 return -ENOMEM;
Florian Mickler2bc43b52009-04-06 22:55:41 +0200153 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
Eric Anholteb014592009-03-10 11:44:52 -0700154 kunmap_atomic(vaddr, KM_USER0);
155
Florian Mickler2bc43b52009-04-06 22:55:41 +0200156 if (unwritten)
157 return -EFAULT;
158
159 return 0;
Eric Anholteb014592009-03-10 11:44:52 -0700160}
161
Eric Anholt280b7132009-03-12 16:56:27 -0700162static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
163{
164 drm_i915_private_t *dev_priv = obj->dev->dev_private;
165 struct drm_i915_gem_object *obj_priv = obj->driver_private;
166
167 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
168 obj_priv->tiling_mode != I915_TILING_NONE;
169}
170
Eric Anholteb014592009-03-10 11:44:52 -0700171static inline int
Eric Anholt40123c12009-03-09 13:42:30 -0700172slow_shmem_copy(struct page *dst_page,
173 int dst_offset,
174 struct page *src_page,
175 int src_offset,
176 int length)
177{
178 char *dst_vaddr, *src_vaddr;
179
180 dst_vaddr = kmap_atomic(dst_page, KM_USER0);
181 if (dst_vaddr == NULL)
182 return -ENOMEM;
183
184 src_vaddr = kmap_atomic(src_page, KM_USER1);
185 if (src_vaddr == NULL) {
186 kunmap_atomic(dst_vaddr, KM_USER0);
187 return -ENOMEM;
188 }
189
190 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
191
192 kunmap_atomic(src_vaddr, KM_USER1);
193 kunmap_atomic(dst_vaddr, KM_USER0);
194
195 return 0;
196}
197
Eric Anholt280b7132009-03-12 16:56:27 -0700198static inline int
199slow_shmem_bit17_copy(struct page *gpu_page,
200 int gpu_offset,
201 struct page *cpu_page,
202 int cpu_offset,
203 int length,
204 int is_read)
205{
206 char *gpu_vaddr, *cpu_vaddr;
207
208 /* Use the unswizzled path if this page isn't affected. */
209 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
210 if (is_read)
211 return slow_shmem_copy(cpu_page, cpu_offset,
212 gpu_page, gpu_offset, length);
213 else
214 return slow_shmem_copy(gpu_page, gpu_offset,
215 cpu_page, cpu_offset, length);
216 }
217
218 gpu_vaddr = kmap_atomic(gpu_page, KM_USER0);
219 if (gpu_vaddr == NULL)
220 return -ENOMEM;
221
222 cpu_vaddr = kmap_atomic(cpu_page, KM_USER1);
223 if (cpu_vaddr == NULL) {
224 kunmap_atomic(gpu_vaddr, KM_USER0);
225 return -ENOMEM;
226 }
227
228 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
229 * XORing with the other bits (A9 for Y, A9 and A10 for X)
230 */
231 while (length > 0) {
232 int cacheline_end = ALIGN(gpu_offset + 1, 64);
233 int this_length = min(cacheline_end - gpu_offset, length);
234 int swizzled_gpu_offset = gpu_offset ^ 64;
235
236 if (is_read) {
237 memcpy(cpu_vaddr + cpu_offset,
238 gpu_vaddr + swizzled_gpu_offset,
239 this_length);
240 } else {
241 memcpy(gpu_vaddr + swizzled_gpu_offset,
242 cpu_vaddr + cpu_offset,
243 this_length);
244 }
245 cpu_offset += this_length;
246 gpu_offset += this_length;
247 length -= this_length;
248 }
249
250 kunmap_atomic(cpu_vaddr, KM_USER1);
251 kunmap_atomic(gpu_vaddr, KM_USER0);
252
253 return 0;
254}
255
Eric Anholt673a3942008-07-30 12:06:12 -0700256/**
Eric Anholteb014592009-03-10 11:44:52 -0700257 * This is the fast shmem pread path, which attempts to copy_from_user directly
258 * from the backing pages of the object to the user's address space. On a
259 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
260 */
261static int
262i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
263 struct drm_i915_gem_pread *args,
264 struct drm_file *file_priv)
265{
266 struct drm_i915_gem_object *obj_priv = obj->driver_private;
267 ssize_t remain;
268 loff_t offset, page_base;
269 char __user *user_data;
270 int page_offset, page_length;
271 int ret;
272
273 user_data = (char __user *) (uintptr_t) args->data_ptr;
274 remain = args->size;
275
276 mutex_lock(&dev->struct_mutex);
277
Chris Wilson4bdadb92010-01-27 13:36:32 +0000278 ret = i915_gem_object_get_pages(obj, 0);
Eric Anholteb014592009-03-10 11:44:52 -0700279 if (ret != 0)
280 goto fail_unlock;
281
282 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
283 args->size);
284 if (ret != 0)
285 goto fail_put_pages;
286
287 obj_priv = obj->driver_private;
288 offset = args->offset;
289
290 while (remain > 0) {
291 /* Operation in this page
292 *
293 * page_base = page offset within aperture
294 * page_offset = offset within page
295 * page_length = bytes to copy for this page
296 */
297 page_base = (offset & ~(PAGE_SIZE-1));
298 page_offset = offset & (PAGE_SIZE-1);
299 page_length = remain;
300 if ((page_offset + remain) > PAGE_SIZE)
301 page_length = PAGE_SIZE - page_offset;
302
303 ret = fast_shmem_read(obj_priv->pages,
304 page_base, page_offset,
305 user_data, page_length);
306 if (ret)
307 goto fail_put_pages;
308
309 remain -= page_length;
310 user_data += page_length;
311 offset += page_length;
312 }
313
314fail_put_pages:
315 i915_gem_object_put_pages(obj);
316fail_unlock:
317 mutex_unlock(&dev->struct_mutex);
318
319 return ret;
320}
321
Chris Wilson07f73f62009-09-14 16:50:30 +0100322static int
323i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
324{
325 int ret;
326
Chris Wilson4bdadb92010-01-27 13:36:32 +0000327 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
Chris Wilson07f73f62009-09-14 16:50:30 +0100328
329 /* If we've insufficient memory to map in the pages, attempt
330 * to make some space by throwing out some old buffers.
331 */
332 if (ret == -ENOMEM) {
333 struct drm_device *dev = obj->dev;
Chris Wilson07f73f62009-09-14 16:50:30 +0100334
335 ret = i915_gem_evict_something(dev, obj->size);
336 if (ret)
337 return ret;
338
Chris Wilson4bdadb92010-01-27 13:36:32 +0000339 ret = i915_gem_object_get_pages(obj, 0);
Chris Wilson07f73f62009-09-14 16:50:30 +0100340 }
341
342 return ret;
343}
344
Eric Anholteb014592009-03-10 11:44:52 -0700345/**
346 * This is the fallback shmem pread path, which allocates temporary storage
347 * in kernel space to copy_to_user into outside of the struct_mutex, so we
348 * can copy out of the object's backing pages while holding the struct mutex
349 * and not take page faults.
350 */
351static int
352i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
353 struct drm_i915_gem_pread *args,
354 struct drm_file *file_priv)
355{
356 struct drm_i915_gem_object *obj_priv = obj->driver_private;
357 struct mm_struct *mm = current->mm;
358 struct page **user_pages;
359 ssize_t remain;
360 loff_t offset, pinned_pages, i;
361 loff_t first_data_page, last_data_page, num_pages;
362 int shmem_page_index, shmem_page_offset;
363 int data_page_index, data_page_offset;
364 int page_length;
365 int ret;
366 uint64_t data_ptr = args->data_ptr;
Eric Anholt280b7132009-03-12 16:56:27 -0700367 int do_bit17_swizzling;
Eric Anholteb014592009-03-10 11:44:52 -0700368
369 remain = args->size;
370
371 /* Pin the user pages containing the data. We can't fault while
372 * holding the struct mutex, yet we want to hold it while
373 * dereferencing the user data.
374 */
375 first_data_page = data_ptr / PAGE_SIZE;
376 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
377 num_pages = last_data_page - first_data_page + 1;
378
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700379 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
Eric Anholteb014592009-03-10 11:44:52 -0700380 if (user_pages == NULL)
381 return -ENOMEM;
382
383 down_read(&mm->mmap_sem);
384 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
Eric Anholte5e9ecd2009-04-07 16:01:22 -0700385 num_pages, 1, 0, user_pages, NULL);
Eric Anholteb014592009-03-10 11:44:52 -0700386 up_read(&mm->mmap_sem);
387 if (pinned_pages < num_pages) {
388 ret = -EFAULT;
389 goto fail_put_user_pages;
390 }
391
Eric Anholt280b7132009-03-12 16:56:27 -0700392 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
393
Eric Anholteb014592009-03-10 11:44:52 -0700394 mutex_lock(&dev->struct_mutex);
395
Chris Wilson07f73f62009-09-14 16:50:30 +0100396 ret = i915_gem_object_get_pages_or_evict(obj);
397 if (ret)
Eric Anholteb014592009-03-10 11:44:52 -0700398 goto fail_unlock;
399
400 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
401 args->size);
402 if (ret != 0)
403 goto fail_put_pages;
404
405 obj_priv = obj->driver_private;
406 offset = args->offset;
407
408 while (remain > 0) {
409 /* Operation in this page
410 *
411 * shmem_page_index = page number within shmem file
412 * shmem_page_offset = offset within page in shmem file
413 * data_page_index = page number in get_user_pages return
414 * data_page_offset = offset with data_page_index page.
415 * page_length = bytes to copy for this page
416 */
417 shmem_page_index = offset / PAGE_SIZE;
418 shmem_page_offset = offset & ~PAGE_MASK;
419 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
420 data_page_offset = data_ptr & ~PAGE_MASK;
421
422 page_length = remain;
423 if ((shmem_page_offset + page_length) > PAGE_SIZE)
424 page_length = PAGE_SIZE - shmem_page_offset;
425 if ((data_page_offset + page_length) > PAGE_SIZE)
426 page_length = PAGE_SIZE - data_page_offset;
427
Eric Anholt280b7132009-03-12 16:56:27 -0700428 if (do_bit17_swizzling) {
429 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
430 shmem_page_offset,
431 user_pages[data_page_index],
432 data_page_offset,
433 page_length,
434 1);
435 } else {
436 ret = slow_shmem_copy(user_pages[data_page_index],
437 data_page_offset,
438 obj_priv->pages[shmem_page_index],
439 shmem_page_offset,
440 page_length);
441 }
Eric Anholteb014592009-03-10 11:44:52 -0700442 if (ret)
443 goto fail_put_pages;
444
445 remain -= page_length;
446 data_ptr += page_length;
447 offset += page_length;
448 }
449
450fail_put_pages:
451 i915_gem_object_put_pages(obj);
452fail_unlock:
453 mutex_unlock(&dev->struct_mutex);
454fail_put_user_pages:
455 for (i = 0; i < pinned_pages; i++) {
456 SetPageDirty(user_pages[i]);
457 page_cache_release(user_pages[i]);
458 }
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700459 drm_free_large(user_pages);
Eric Anholteb014592009-03-10 11:44:52 -0700460
461 return ret;
462}
463
Eric Anholt673a3942008-07-30 12:06:12 -0700464/**
465 * Reads data from the object referenced by handle.
466 *
467 * On error, the contents of *data are undefined.
468 */
469int
470i915_gem_pread_ioctl(struct drm_device *dev, void *data,
471 struct drm_file *file_priv)
472{
473 struct drm_i915_gem_pread *args = data;
474 struct drm_gem_object *obj;
475 struct drm_i915_gem_object *obj_priv;
Eric Anholt673a3942008-07-30 12:06:12 -0700476 int ret;
477
478 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
479 if (obj == NULL)
480 return -EBADF;
481 obj_priv = obj->driver_private;
482
483 /* Bounds check source.
484 *
485 * XXX: This could use review for overflow issues...
486 */
487 if (args->offset > obj->size || args->size > obj->size ||
488 args->offset + args->size > obj->size) {
Luca Barbieribc9025b2010-02-09 05:49:12 +0000489 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700490 return -EINVAL;
491 }
492
Eric Anholt280b7132009-03-12 16:56:27 -0700493 if (i915_gem_object_needs_bit17_swizzle(obj)) {
Eric Anholteb014592009-03-10 11:44:52 -0700494 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
Eric Anholt280b7132009-03-12 16:56:27 -0700495 } else {
496 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
497 if (ret != 0)
498 ret = i915_gem_shmem_pread_slow(dev, obj, args,
499 file_priv);
500 }
Eric Anholt673a3942008-07-30 12:06:12 -0700501
Luca Barbieribc9025b2010-02-09 05:49:12 +0000502 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700503
Eric Anholteb014592009-03-10 11:44:52 -0700504 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700505}
506
Keith Packard0839ccb2008-10-30 19:38:48 -0700507/* This is the fast write path which cannot handle
508 * page faults in the source data
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700509 */
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700510
Keith Packard0839ccb2008-10-30 19:38:48 -0700511static inline int
512fast_user_write(struct io_mapping *mapping,
513 loff_t page_base, int page_offset,
514 char __user *user_data,
515 int length)
516{
517 char *vaddr_atomic;
518 unsigned long unwritten;
519
520 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
521 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
522 user_data, length);
523 io_mapping_unmap_atomic(vaddr_atomic);
524 if (unwritten)
525 return -EFAULT;
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700526 return 0;
Keith Packard0839ccb2008-10-30 19:38:48 -0700527}
528
529/* Here's the write path which can sleep for
530 * page faults
531 */
532
533static inline int
Eric Anholt3de09aa2009-03-09 09:42:23 -0700534slow_kernel_write(struct io_mapping *mapping,
535 loff_t gtt_base, int gtt_offset,
536 struct page *user_page, int user_offset,
537 int length)
Keith Packard0839ccb2008-10-30 19:38:48 -0700538{
Eric Anholt3de09aa2009-03-09 09:42:23 -0700539 char *src_vaddr, *dst_vaddr;
Keith Packard0839ccb2008-10-30 19:38:48 -0700540 unsigned long unwritten;
541
Eric Anholt3de09aa2009-03-09 09:42:23 -0700542 dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
543 src_vaddr = kmap_atomic(user_page, KM_USER1);
544 unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
545 src_vaddr + user_offset,
546 length);
547 kunmap_atomic(src_vaddr, KM_USER1);
548 io_mapping_unmap_atomic(dst_vaddr);
Keith Packard0839ccb2008-10-30 19:38:48 -0700549 if (unwritten)
550 return -EFAULT;
551 return 0;
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700552}
553
Eric Anholt40123c12009-03-09 13:42:30 -0700554static inline int
555fast_shmem_write(struct page **pages,
556 loff_t page_base, int page_offset,
557 char __user *data,
558 int length)
559{
560 char __iomem *vaddr;
Dave Airlied0088772009-03-28 20:29:48 -0400561 unsigned long unwritten;
Eric Anholt40123c12009-03-09 13:42:30 -0700562
563 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
564 if (vaddr == NULL)
565 return -ENOMEM;
Dave Airlied0088772009-03-28 20:29:48 -0400566 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
Eric Anholt40123c12009-03-09 13:42:30 -0700567 kunmap_atomic(vaddr, KM_USER0);
568
Dave Airlied0088772009-03-28 20:29:48 -0400569 if (unwritten)
570 return -EFAULT;
Eric Anholt40123c12009-03-09 13:42:30 -0700571 return 0;
572}
573
Eric Anholt3de09aa2009-03-09 09:42:23 -0700574/**
575 * This is the fast pwrite path, where we copy the data directly from the
576 * user into the GTT, uncached.
577 */
Eric Anholt673a3942008-07-30 12:06:12 -0700578static int
Eric Anholt3de09aa2009-03-09 09:42:23 -0700579i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
580 struct drm_i915_gem_pwrite *args,
581 struct drm_file *file_priv)
Eric Anholt673a3942008-07-30 12:06:12 -0700582{
583 struct drm_i915_gem_object *obj_priv = obj->driver_private;
Keith Packard0839ccb2008-10-30 19:38:48 -0700584 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -0700585 ssize_t remain;
Keith Packard0839ccb2008-10-30 19:38:48 -0700586 loff_t offset, page_base;
Eric Anholt673a3942008-07-30 12:06:12 -0700587 char __user *user_data;
Keith Packard0839ccb2008-10-30 19:38:48 -0700588 int page_offset, page_length;
589 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700590
591 user_data = (char __user *) (uintptr_t) args->data_ptr;
592 remain = args->size;
593 if (!access_ok(VERIFY_READ, user_data, remain))
594 return -EFAULT;
595
596
597 mutex_lock(&dev->struct_mutex);
598 ret = i915_gem_object_pin(obj, 0);
599 if (ret) {
600 mutex_unlock(&dev->struct_mutex);
601 return ret;
602 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800603 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
Eric Anholt673a3942008-07-30 12:06:12 -0700604 if (ret)
605 goto fail;
606
607 obj_priv = obj->driver_private;
608 offset = obj_priv->gtt_offset + args->offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700609
610 while (remain > 0) {
611 /* Operation in this page
612 *
Keith Packard0839ccb2008-10-30 19:38:48 -0700613 * page_base = page offset within aperture
614 * page_offset = offset within page
615 * page_length = bytes to copy for this page
Eric Anholt673a3942008-07-30 12:06:12 -0700616 */
Keith Packard0839ccb2008-10-30 19:38:48 -0700617 page_base = (offset & ~(PAGE_SIZE-1));
618 page_offset = offset & (PAGE_SIZE-1);
619 page_length = remain;
620 if ((page_offset + remain) > PAGE_SIZE)
621 page_length = PAGE_SIZE - page_offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700622
Keith Packard0839ccb2008-10-30 19:38:48 -0700623 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
624 page_offset, user_data, page_length);
Eric Anholt673a3942008-07-30 12:06:12 -0700625
Keith Packard0839ccb2008-10-30 19:38:48 -0700626 /* If we get a fault while copying data, then (presumably) our
Eric Anholt3de09aa2009-03-09 09:42:23 -0700627 * source page isn't available. Return the error and we'll
628 * retry in the slow path.
Keith Packard0839ccb2008-10-30 19:38:48 -0700629 */
Eric Anholt3de09aa2009-03-09 09:42:23 -0700630 if (ret)
631 goto fail;
Eric Anholt673a3942008-07-30 12:06:12 -0700632
Keith Packard0839ccb2008-10-30 19:38:48 -0700633 remain -= page_length;
634 user_data += page_length;
635 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700636 }
Eric Anholt673a3942008-07-30 12:06:12 -0700637
638fail:
639 i915_gem_object_unpin(obj);
640 mutex_unlock(&dev->struct_mutex);
641
642 return ret;
643}
644
Eric Anholt3de09aa2009-03-09 09:42:23 -0700645/**
646 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
647 * the memory and maps it using kmap_atomic for copying.
648 *
649 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
650 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
651 */
Eric Anholt3043c602008-10-02 12:24:47 -0700652static int
Eric Anholt3de09aa2009-03-09 09:42:23 -0700653i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
654 struct drm_i915_gem_pwrite *args,
655 struct drm_file *file_priv)
Eric Anholt673a3942008-07-30 12:06:12 -0700656{
Eric Anholt3de09aa2009-03-09 09:42:23 -0700657 struct drm_i915_gem_object *obj_priv = obj->driver_private;
658 drm_i915_private_t *dev_priv = dev->dev_private;
659 ssize_t remain;
660 loff_t gtt_page_base, offset;
661 loff_t first_data_page, last_data_page, num_pages;
662 loff_t pinned_pages, i;
663 struct page **user_pages;
664 struct mm_struct *mm = current->mm;
665 int gtt_page_offset, data_page_offset, data_page_index, page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700666 int ret;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700667 uint64_t data_ptr = args->data_ptr;
668
669 remain = args->size;
670
671 /* Pin the user pages containing the data. We can't fault while
672 * holding the struct mutex, and all of the pwrite implementations
673 * want to hold it while dereferencing the user data.
674 */
675 first_data_page = data_ptr / PAGE_SIZE;
676 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
677 num_pages = last_data_page - first_data_page + 1;
678
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700679 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
Eric Anholt3de09aa2009-03-09 09:42:23 -0700680 if (user_pages == NULL)
681 return -ENOMEM;
682
683 down_read(&mm->mmap_sem);
684 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
685 num_pages, 0, 0, user_pages, NULL);
686 up_read(&mm->mmap_sem);
687 if (pinned_pages < num_pages) {
688 ret = -EFAULT;
689 goto out_unpin_pages;
690 }
691
692 mutex_lock(&dev->struct_mutex);
693 ret = i915_gem_object_pin(obj, 0);
694 if (ret)
695 goto out_unlock;
696
697 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
698 if (ret)
699 goto out_unpin_object;
700
701 obj_priv = obj->driver_private;
702 offset = obj_priv->gtt_offset + args->offset;
703
704 while (remain > 0) {
705 /* Operation in this page
706 *
707 * gtt_page_base = page offset within aperture
708 * gtt_page_offset = offset within page in aperture
709 * data_page_index = page number in get_user_pages return
710 * data_page_offset = offset with data_page_index page.
711 * page_length = bytes to copy for this page
712 */
713 gtt_page_base = offset & PAGE_MASK;
714 gtt_page_offset = offset & ~PAGE_MASK;
715 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
716 data_page_offset = data_ptr & ~PAGE_MASK;
717
718 page_length = remain;
719 if ((gtt_page_offset + page_length) > PAGE_SIZE)
720 page_length = PAGE_SIZE - gtt_page_offset;
721 if ((data_page_offset + page_length) > PAGE_SIZE)
722 page_length = PAGE_SIZE - data_page_offset;
723
724 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
725 gtt_page_base, gtt_page_offset,
726 user_pages[data_page_index],
727 data_page_offset,
728 page_length);
729
730 /* If we get a fault while copying data, then (presumably) our
731 * source page isn't available. Return the error and we'll
732 * retry in the slow path.
733 */
734 if (ret)
735 goto out_unpin_object;
736
737 remain -= page_length;
738 offset += page_length;
739 data_ptr += page_length;
740 }
741
742out_unpin_object:
743 i915_gem_object_unpin(obj);
744out_unlock:
745 mutex_unlock(&dev->struct_mutex);
746out_unpin_pages:
747 for (i = 0; i < pinned_pages; i++)
748 page_cache_release(user_pages[i]);
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700749 drm_free_large(user_pages);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700750
751 return ret;
752}
753
Eric Anholt40123c12009-03-09 13:42:30 -0700754/**
755 * This is the fast shmem pwrite path, which attempts to directly
756 * copy_from_user into the kmapped pages backing the object.
757 */
Eric Anholt673a3942008-07-30 12:06:12 -0700758static int
Eric Anholt40123c12009-03-09 13:42:30 -0700759i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
760 struct drm_i915_gem_pwrite *args,
761 struct drm_file *file_priv)
Eric Anholt673a3942008-07-30 12:06:12 -0700762{
Eric Anholt40123c12009-03-09 13:42:30 -0700763 struct drm_i915_gem_object *obj_priv = obj->driver_private;
764 ssize_t remain;
765 loff_t offset, page_base;
766 char __user *user_data;
767 int page_offset, page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700768 int ret;
Eric Anholt40123c12009-03-09 13:42:30 -0700769
770 user_data = (char __user *) (uintptr_t) args->data_ptr;
771 remain = args->size;
Eric Anholt673a3942008-07-30 12:06:12 -0700772
773 mutex_lock(&dev->struct_mutex);
774
Chris Wilson4bdadb92010-01-27 13:36:32 +0000775 ret = i915_gem_object_get_pages(obj, 0);
Eric Anholt40123c12009-03-09 13:42:30 -0700776 if (ret != 0)
777 goto fail_unlock;
778
Eric Anholte47c68e2008-11-14 13:35:19 -0800779 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
Eric Anholt40123c12009-03-09 13:42:30 -0700780 if (ret != 0)
781 goto fail_put_pages;
Eric Anholt673a3942008-07-30 12:06:12 -0700782
Eric Anholt40123c12009-03-09 13:42:30 -0700783 obj_priv = obj->driver_private;
Eric Anholt673a3942008-07-30 12:06:12 -0700784 offset = args->offset;
Eric Anholt40123c12009-03-09 13:42:30 -0700785 obj_priv->dirty = 1;
Eric Anholt673a3942008-07-30 12:06:12 -0700786
Eric Anholt40123c12009-03-09 13:42:30 -0700787 while (remain > 0) {
788 /* Operation in this page
789 *
790 * page_base = page offset within aperture
791 * page_offset = offset within page
792 * page_length = bytes to copy for this page
793 */
794 page_base = (offset & ~(PAGE_SIZE-1));
795 page_offset = offset & (PAGE_SIZE-1);
796 page_length = remain;
797 if ((page_offset + remain) > PAGE_SIZE)
798 page_length = PAGE_SIZE - page_offset;
799
800 ret = fast_shmem_write(obj_priv->pages,
801 page_base, page_offset,
802 user_data, page_length);
803 if (ret)
804 goto fail_put_pages;
805
806 remain -= page_length;
807 user_data += page_length;
808 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700809 }
810
Eric Anholt40123c12009-03-09 13:42:30 -0700811fail_put_pages:
812 i915_gem_object_put_pages(obj);
813fail_unlock:
Eric Anholt673a3942008-07-30 12:06:12 -0700814 mutex_unlock(&dev->struct_mutex);
815
Eric Anholt40123c12009-03-09 13:42:30 -0700816 return ret;
817}
818
819/**
820 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
821 * the memory and maps it using kmap_atomic for copying.
822 *
823 * This avoids taking mmap_sem for faulting on the user's address while the
824 * struct_mutex is held.
825 */
826static int
827i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
828 struct drm_i915_gem_pwrite *args,
829 struct drm_file *file_priv)
830{
831 struct drm_i915_gem_object *obj_priv = obj->driver_private;
832 struct mm_struct *mm = current->mm;
833 struct page **user_pages;
834 ssize_t remain;
835 loff_t offset, pinned_pages, i;
836 loff_t first_data_page, last_data_page, num_pages;
837 int shmem_page_index, shmem_page_offset;
838 int data_page_index, data_page_offset;
839 int page_length;
840 int ret;
841 uint64_t data_ptr = args->data_ptr;
Eric Anholt280b7132009-03-12 16:56:27 -0700842 int do_bit17_swizzling;
Eric Anholt40123c12009-03-09 13:42:30 -0700843
844 remain = args->size;
845
846 /* Pin the user pages containing the data. We can't fault while
847 * holding the struct mutex, and all of the pwrite implementations
848 * want to hold it while dereferencing the user data.
849 */
850 first_data_page = data_ptr / PAGE_SIZE;
851 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
852 num_pages = last_data_page - first_data_page + 1;
853
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700854 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
Eric Anholt40123c12009-03-09 13:42:30 -0700855 if (user_pages == NULL)
856 return -ENOMEM;
857
858 down_read(&mm->mmap_sem);
859 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
860 num_pages, 0, 0, user_pages, NULL);
861 up_read(&mm->mmap_sem);
862 if (pinned_pages < num_pages) {
863 ret = -EFAULT;
864 goto fail_put_user_pages;
865 }
866
Eric Anholt280b7132009-03-12 16:56:27 -0700867 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
868
Eric Anholt40123c12009-03-09 13:42:30 -0700869 mutex_lock(&dev->struct_mutex);
870
Chris Wilson07f73f62009-09-14 16:50:30 +0100871 ret = i915_gem_object_get_pages_or_evict(obj);
872 if (ret)
Eric Anholt40123c12009-03-09 13:42:30 -0700873 goto fail_unlock;
874
875 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
876 if (ret != 0)
877 goto fail_put_pages;
878
879 obj_priv = obj->driver_private;
880 offset = args->offset;
881 obj_priv->dirty = 1;
882
883 while (remain > 0) {
884 /* Operation in this page
885 *
886 * shmem_page_index = page number within shmem file
887 * shmem_page_offset = offset within page in shmem file
888 * data_page_index = page number in get_user_pages return
889 * data_page_offset = offset with data_page_index page.
890 * page_length = bytes to copy for this page
891 */
892 shmem_page_index = offset / PAGE_SIZE;
893 shmem_page_offset = offset & ~PAGE_MASK;
894 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
895 data_page_offset = data_ptr & ~PAGE_MASK;
896
897 page_length = remain;
898 if ((shmem_page_offset + page_length) > PAGE_SIZE)
899 page_length = PAGE_SIZE - shmem_page_offset;
900 if ((data_page_offset + page_length) > PAGE_SIZE)
901 page_length = PAGE_SIZE - data_page_offset;
902
Eric Anholt280b7132009-03-12 16:56:27 -0700903 if (do_bit17_swizzling) {
904 ret = slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
905 shmem_page_offset,
906 user_pages[data_page_index],
907 data_page_offset,
908 page_length,
909 0);
910 } else {
911 ret = slow_shmem_copy(obj_priv->pages[shmem_page_index],
912 shmem_page_offset,
913 user_pages[data_page_index],
914 data_page_offset,
915 page_length);
916 }
Eric Anholt40123c12009-03-09 13:42:30 -0700917 if (ret)
918 goto fail_put_pages;
919
920 remain -= page_length;
921 data_ptr += page_length;
922 offset += page_length;
923 }
924
925fail_put_pages:
926 i915_gem_object_put_pages(obj);
927fail_unlock:
928 mutex_unlock(&dev->struct_mutex);
929fail_put_user_pages:
930 for (i = 0; i < pinned_pages; i++)
931 page_cache_release(user_pages[i]);
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700932 drm_free_large(user_pages);
Eric Anholt40123c12009-03-09 13:42:30 -0700933
934 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700935}
936
937/**
938 * Writes data to the object referenced by handle.
939 *
940 * On error, the contents of the buffer that were to be modified are undefined.
941 */
942int
943i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
944 struct drm_file *file_priv)
945{
946 struct drm_i915_gem_pwrite *args = data;
947 struct drm_gem_object *obj;
948 struct drm_i915_gem_object *obj_priv;
949 int ret = 0;
950
951 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
952 if (obj == NULL)
953 return -EBADF;
954 obj_priv = obj->driver_private;
955
956 /* Bounds check destination.
957 *
958 * XXX: This could use review for overflow issues...
959 */
960 if (args->offset > obj->size || args->size > obj->size ||
961 args->offset + args->size > obj->size) {
Luca Barbieribc9025b2010-02-09 05:49:12 +0000962 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700963 return -EINVAL;
964 }
965
966 /* We can only do the GTT pwrite on untiled buffers, as otherwise
967 * it would end up going through the fenced access, and we'll get
968 * different detiling behavior between reading and writing.
969 * pread/pwrite currently are reading and writing from the CPU
970 * perspective, requiring manual detiling by the client.
971 */
Dave Airlie71acb5e2008-12-30 20:31:46 +1000972 if (obj_priv->phys_obj)
973 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
974 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
Eric Anholt3de09aa2009-03-09 09:42:23 -0700975 dev->gtt_total != 0) {
976 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
977 if (ret == -EFAULT) {
978 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
979 file_priv);
980 }
Eric Anholt280b7132009-03-12 16:56:27 -0700981 } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
982 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
Eric Anholt40123c12009-03-09 13:42:30 -0700983 } else {
984 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
985 if (ret == -EFAULT) {
986 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
987 file_priv);
988 }
989 }
Eric Anholt673a3942008-07-30 12:06:12 -0700990
991#if WATCH_PWRITE
992 if (ret)
993 DRM_INFO("pwrite failed %d\n", ret);
994#endif
995
Luca Barbieribc9025b2010-02-09 05:49:12 +0000996 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700997
998 return ret;
999}
1000
1001/**
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001002 * Called when user space prepares to use an object with the CPU, either
1003 * through the mmap ioctl's mapping or a GTT mapping.
Eric Anholt673a3942008-07-30 12:06:12 -07001004 */
1005int
1006i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1007 struct drm_file *file_priv)
1008{
Eric Anholta09ba7f2009-08-29 12:49:51 -07001009 struct drm_i915_private *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07001010 struct drm_i915_gem_set_domain *args = data;
1011 struct drm_gem_object *obj;
Jesse Barnes652c3932009-08-17 13:31:43 -07001012 struct drm_i915_gem_object *obj_priv;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001013 uint32_t read_domains = args->read_domains;
1014 uint32_t write_domain = args->write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -07001015 int ret;
1016
1017 if (!(dev->driver->driver_features & DRIVER_GEM))
1018 return -ENODEV;
1019
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001020 /* Only handle setting domains to types used by the CPU. */
Chris Wilson21d509e2009-06-06 09:46:02 +01001021 if (write_domain & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001022 return -EINVAL;
1023
Chris Wilson21d509e2009-06-06 09:46:02 +01001024 if (read_domains & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001025 return -EINVAL;
1026
1027 /* Having something in the write domain implies it's in the read
1028 * domain, and only that read domain. Enforce that in the request.
1029 */
1030 if (write_domain != 0 && read_domains != write_domain)
1031 return -EINVAL;
1032
Eric Anholt673a3942008-07-30 12:06:12 -07001033 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1034 if (obj == NULL)
1035 return -EBADF;
Jesse Barnes652c3932009-08-17 13:31:43 -07001036 obj_priv = obj->driver_private;
Eric Anholt673a3942008-07-30 12:06:12 -07001037
1038 mutex_lock(&dev->struct_mutex);
Jesse Barnes652c3932009-08-17 13:31:43 -07001039
1040 intel_mark_busy(dev, obj);
1041
Eric Anholt673a3942008-07-30 12:06:12 -07001042#if WATCH_BUF
Krzysztof Halasacfd43c02009-06-20 00:31:28 +02001043 DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001044 obj, obj->size, read_domains, write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07001045#endif
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001046 if (read_domains & I915_GEM_DOMAIN_GTT) {
1047 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
Eric Anholt02354392008-11-26 13:58:13 -08001048
Eric Anholta09ba7f2009-08-29 12:49:51 -07001049 /* Update the LRU on the fence for the CPU access that's
1050 * about to occur.
1051 */
1052 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1053 list_move_tail(&obj_priv->fence_list,
1054 &dev_priv->mm.fence_list);
1055 }
1056
Eric Anholt02354392008-11-26 13:58:13 -08001057 /* Silently promote "you're not bound, there was nothing to do"
1058 * to success, since the client was just asking us to
1059 * make sure everything was done.
1060 */
1061 if (ret == -EINVAL)
1062 ret = 0;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001063 } else {
Eric Anholte47c68e2008-11-14 13:35:19 -08001064 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001065 }
1066
Eric Anholt673a3942008-07-30 12:06:12 -07001067 drm_gem_object_unreference(obj);
1068 mutex_unlock(&dev->struct_mutex);
1069 return ret;
1070}
1071
1072/**
1073 * Called when user space has done writes to this buffer
1074 */
1075int
1076i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1077 struct drm_file *file_priv)
1078{
1079 struct drm_i915_gem_sw_finish *args = data;
1080 struct drm_gem_object *obj;
1081 struct drm_i915_gem_object *obj_priv;
1082 int ret = 0;
1083
1084 if (!(dev->driver->driver_features & DRIVER_GEM))
1085 return -ENODEV;
1086
1087 mutex_lock(&dev->struct_mutex);
1088 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1089 if (obj == NULL) {
1090 mutex_unlock(&dev->struct_mutex);
1091 return -EBADF;
1092 }
1093
1094#if WATCH_BUF
Krzysztof Halasacfd43c02009-06-20 00:31:28 +02001095 DRM_INFO("%s: sw_finish %d (%p %zd)\n",
Eric Anholt673a3942008-07-30 12:06:12 -07001096 __func__, args->handle, obj, obj->size);
1097#endif
1098 obj_priv = obj->driver_private;
1099
1100 /* Pinned buffers may be scanout, so flush the cache */
Eric Anholte47c68e2008-11-14 13:35:19 -08001101 if (obj_priv->pin_count)
1102 i915_gem_object_flush_cpu_write_domain(obj);
1103
Eric Anholt673a3942008-07-30 12:06:12 -07001104 drm_gem_object_unreference(obj);
1105 mutex_unlock(&dev->struct_mutex);
1106 return ret;
1107}
1108
1109/**
1110 * Maps the contents of an object, returning the address it is mapped
1111 * into.
1112 *
1113 * While the mapping holds a reference on the contents of the object, it doesn't
1114 * imply a ref on the object itself.
1115 */
1116int
1117i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1118 struct drm_file *file_priv)
1119{
1120 struct drm_i915_gem_mmap *args = data;
1121 struct drm_gem_object *obj;
1122 loff_t offset;
1123 unsigned long addr;
1124
1125 if (!(dev->driver->driver_features & DRIVER_GEM))
1126 return -ENODEV;
1127
1128 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1129 if (obj == NULL)
1130 return -EBADF;
1131
1132 offset = args->offset;
1133
1134 down_write(&current->mm->mmap_sem);
1135 addr = do_mmap(obj->filp, 0, args->size,
1136 PROT_READ | PROT_WRITE, MAP_SHARED,
1137 args->offset);
1138 up_write(&current->mm->mmap_sem);
Luca Barbieribc9025b2010-02-09 05:49:12 +00001139 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001140 if (IS_ERR((void *)addr))
1141 return addr;
1142
1143 args->addr_ptr = (uint64_t) addr;
1144
1145 return 0;
1146}
1147
Jesse Barnesde151cf2008-11-12 10:03:55 -08001148/**
1149 * i915_gem_fault - fault a page into the GTT
1150 * vma: VMA in question
1151 * vmf: fault info
1152 *
1153 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1154 * from userspace. The fault handler takes care of binding the object to
1155 * the GTT (if needed), allocating and programming a fence register (again,
1156 * only if needed based on whether the old reg is still valid or the object
1157 * is tiled) and inserting a new PTE into the faulting process.
1158 *
1159 * Note that the faulting process may involve evicting existing objects
1160 * from the GTT and/or fence registers to make room. So performance may
1161 * suffer if the GTT working set is large or there are few fence registers
1162 * left.
1163 */
1164int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1165{
1166 struct drm_gem_object *obj = vma->vm_private_data;
1167 struct drm_device *dev = obj->dev;
1168 struct drm_i915_private *dev_priv = dev->dev_private;
1169 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1170 pgoff_t page_offset;
1171 unsigned long pfn;
1172 int ret = 0;
Jesse Barnes0f973f22009-01-26 17:10:45 -08001173 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001174
1175 /* We don't use vmf->pgoff since that has the fake offset */
1176 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1177 PAGE_SHIFT;
1178
1179 /* Now bind it into the GTT if needed */
1180 mutex_lock(&dev->struct_mutex);
1181 if (!obj_priv->gtt_space) {
Chris Wilsone67b8ce2009-09-14 16:50:26 +01001182 ret = i915_gem_object_bind_to_gtt(obj, 0);
Chris Wilsonc7150892009-09-23 00:43:56 +01001183 if (ret)
1184 goto unlock;
Kristian Høgsberg07f4f3e2009-05-27 14:37:28 -04001185
Jesse Barnes14b60392009-05-20 16:47:08 -04001186 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001187
1188 ret = i915_gem_object_set_to_gtt_domain(obj, write);
Chris Wilsonc7150892009-09-23 00:43:56 +01001189 if (ret)
1190 goto unlock;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001191 }
1192
1193 /* Need a new fence register? */
Eric Anholta09ba7f2009-08-29 12:49:51 -07001194 if (obj_priv->tiling_mode != I915_TILING_NONE) {
Chris Wilson8c4b8c32009-06-17 22:08:52 +01001195 ret = i915_gem_object_get_fence_reg(obj);
Chris Wilsonc7150892009-09-23 00:43:56 +01001196 if (ret)
1197 goto unlock;
Eric Anholtd9ddcb92009-01-27 10:33:49 -08001198 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08001199
1200 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1201 page_offset;
1202
1203 /* Finally, remap it using the new GTT offset */
1204 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
Chris Wilsonc7150892009-09-23 00:43:56 +01001205unlock:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001206 mutex_unlock(&dev->struct_mutex);
1207
1208 switch (ret) {
Chris Wilsonc7150892009-09-23 00:43:56 +01001209 case 0:
1210 case -ERESTARTSYS:
1211 return VM_FAULT_NOPAGE;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001212 case -ENOMEM:
1213 case -EAGAIN:
1214 return VM_FAULT_OOM;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001215 default:
Chris Wilsonc7150892009-09-23 00:43:56 +01001216 return VM_FAULT_SIGBUS;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001217 }
1218}
1219
1220/**
1221 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1222 * @obj: obj in question
1223 *
1224 * GEM memory mapping works by handing back to userspace a fake mmap offset
1225 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1226 * up the object based on the offset and sets up the various memory mapping
1227 * structures.
1228 *
1229 * This routine allocates and attaches a fake offset for @obj.
1230 */
1231static int
1232i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1233{
1234 struct drm_device *dev = obj->dev;
1235 struct drm_gem_mm *mm = dev->mm_private;
1236 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1237 struct drm_map_list *list;
Benjamin Herrenschmidtf77d3902009-02-02 16:55:46 +11001238 struct drm_local_map *map;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001239 int ret = 0;
1240
1241 /* Set the object up for mmap'ing */
1242 list = &obj->map_list;
Eric Anholt9a298b22009-03-24 12:23:04 -07001243 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001244 if (!list->map)
1245 return -ENOMEM;
1246
1247 map = list->map;
1248 map->type = _DRM_GEM;
1249 map->size = obj->size;
1250 map->handle = obj;
1251
1252 /* Get a DRM GEM mmap offset allocated... */
1253 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1254 obj->size / PAGE_SIZE, 0, 0);
1255 if (!list->file_offset_node) {
1256 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1257 ret = -ENOMEM;
1258 goto out_free_list;
1259 }
1260
1261 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1262 obj->size / PAGE_SIZE, 0);
1263 if (!list->file_offset_node) {
1264 ret = -ENOMEM;
1265 goto out_free_list;
1266 }
1267
1268 list->hash.key = list->file_offset_node->start;
1269 if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1270 DRM_ERROR("failed to add to map hash\n");
Chris Wilson5618ca62009-12-02 15:15:30 +00001271 ret = -ENOMEM;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001272 goto out_free_mm;
1273 }
1274
1275 /* By now we should be all set, any drm_mmap request on the offset
1276 * below will get to our mmap & fault handler */
1277 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1278
1279 return 0;
1280
1281out_free_mm:
1282 drm_mm_put_block(list->file_offset_node);
1283out_free_list:
Eric Anholt9a298b22009-03-24 12:23:04 -07001284 kfree(list->map);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001285
1286 return ret;
1287}
1288
Chris Wilson901782b2009-07-10 08:18:50 +01001289/**
1290 * i915_gem_release_mmap - remove physical page mappings
1291 * @obj: obj in question
1292 *
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02001293 * Preserve the reservation of the mmapping with the DRM core code, but
Chris Wilson901782b2009-07-10 08:18:50 +01001294 * relinquish ownership of the pages back to the system.
1295 *
1296 * It is vital that we remove the page mapping if we have mapped a tiled
1297 * object through the GTT and then lose the fence register due to
1298 * resource pressure. Similarly if the object has been moved out of the
1299 * aperture, than pages mapped into userspace must be revoked. Removing the
1300 * mapping will then trigger a page fault on the next user access, allowing
1301 * fixup by i915_gem_fault().
1302 */
Eric Anholtd05ca302009-07-10 13:02:26 -07001303void
Chris Wilson901782b2009-07-10 08:18:50 +01001304i915_gem_release_mmap(struct drm_gem_object *obj)
1305{
1306 struct drm_device *dev = obj->dev;
1307 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1308
1309 if (dev->dev_mapping)
1310 unmap_mapping_range(dev->dev_mapping,
1311 obj_priv->mmap_offset, obj->size, 1);
1312}
1313
Jesse Barnesab00b3e2009-02-11 14:01:46 -08001314static void
1315i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1316{
1317 struct drm_device *dev = obj->dev;
1318 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1319 struct drm_gem_mm *mm = dev->mm_private;
1320 struct drm_map_list *list;
1321
1322 list = &obj->map_list;
1323 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1324
1325 if (list->file_offset_node) {
1326 drm_mm_put_block(list->file_offset_node);
1327 list->file_offset_node = NULL;
1328 }
1329
1330 if (list->map) {
Eric Anholt9a298b22009-03-24 12:23:04 -07001331 kfree(list->map);
Jesse Barnesab00b3e2009-02-11 14:01:46 -08001332 list->map = NULL;
1333 }
1334
1335 obj_priv->mmap_offset = 0;
1336}
1337
Jesse Barnesde151cf2008-11-12 10:03:55 -08001338/**
1339 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1340 * @obj: object to check
1341 *
1342 * Return the required GTT alignment for an object, taking into account
1343 * potential fence register mapping if needed.
1344 */
1345static uint32_t
1346i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1347{
1348 struct drm_device *dev = obj->dev;
1349 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1350 int start, i;
1351
1352 /*
1353 * Minimum alignment is 4k (GTT page size), but might be greater
1354 * if a fence register is needed for the object.
1355 */
1356 if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1357 return 4096;
1358
1359 /*
1360 * Previous chips need to be aligned to the size of the smallest
1361 * fence register that can contain the object.
1362 */
1363 if (IS_I9XX(dev))
1364 start = 1024*1024;
1365 else
1366 start = 512*1024;
1367
1368 for (i = start; i < obj->size; i <<= 1)
1369 ;
1370
1371 return i;
1372}
1373
1374/**
1375 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1376 * @dev: DRM device
1377 * @data: GTT mapping ioctl data
1378 * @file_priv: GEM object info
1379 *
1380 * Simply returns the fake offset to userspace so it can mmap it.
1381 * The mmap call will end up in drm_gem_mmap(), which will set things
1382 * up so we can get faults in the handler above.
1383 *
1384 * The fault handler will take care of binding the object into the GTT
1385 * (since it may have been evicted to make room for something), allocating
1386 * a fence register, and mapping the appropriate aperture address into
1387 * userspace.
1388 */
1389int
1390i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1391 struct drm_file *file_priv)
1392{
1393 struct drm_i915_gem_mmap_gtt *args = data;
1394 struct drm_i915_private *dev_priv = dev->dev_private;
1395 struct drm_gem_object *obj;
1396 struct drm_i915_gem_object *obj_priv;
1397 int ret;
1398
1399 if (!(dev->driver->driver_features & DRIVER_GEM))
1400 return -ENODEV;
1401
1402 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1403 if (obj == NULL)
1404 return -EBADF;
1405
1406 mutex_lock(&dev->struct_mutex);
1407
1408 obj_priv = obj->driver_private;
1409
Chris Wilsonab182822009-09-22 18:46:17 +01001410 if (obj_priv->madv != I915_MADV_WILLNEED) {
1411 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1412 drm_gem_object_unreference(obj);
1413 mutex_unlock(&dev->struct_mutex);
1414 return -EINVAL;
1415 }
1416
1417
Jesse Barnesde151cf2008-11-12 10:03:55 -08001418 if (!obj_priv->mmap_offset) {
1419 ret = i915_gem_create_mmap_offset(obj);
Chris Wilson13af1062009-02-11 14:26:31 +00001420 if (ret) {
1421 drm_gem_object_unreference(obj);
1422 mutex_unlock(&dev->struct_mutex);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001423 return ret;
Chris Wilson13af1062009-02-11 14:26:31 +00001424 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08001425 }
1426
1427 args->offset = obj_priv->mmap_offset;
1428
Jesse Barnesde151cf2008-11-12 10:03:55 -08001429 /*
1430 * Pull it into the GTT so that we have a page list (makes the
1431 * initial fault faster and any subsequent flushing possible).
1432 */
1433 if (!obj_priv->agp_mem) {
Chris Wilsone67b8ce2009-09-14 16:50:26 +01001434 ret = i915_gem_object_bind_to_gtt(obj, 0);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001435 if (ret) {
1436 drm_gem_object_unreference(obj);
1437 mutex_unlock(&dev->struct_mutex);
1438 return ret;
1439 }
Jesse Barnes14b60392009-05-20 16:47:08 -04001440 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001441 }
1442
1443 drm_gem_object_unreference(obj);
1444 mutex_unlock(&dev->struct_mutex);
1445
1446 return 0;
1447}
1448
Ben Gamari6911a9b2009-04-02 11:24:54 -07001449void
Eric Anholt856fa192009-03-19 14:10:50 -07001450i915_gem_object_put_pages(struct drm_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07001451{
1452 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1453 int page_count = obj->size / PAGE_SIZE;
1454 int i;
1455
Eric Anholt856fa192009-03-19 14:10:50 -07001456 BUG_ON(obj_priv->pages_refcount == 0);
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001457 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
Eric Anholt856fa192009-03-19 14:10:50 -07001458
1459 if (--obj_priv->pages_refcount != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07001460 return;
1461
Eric Anholt280b7132009-03-12 16:56:27 -07001462 if (obj_priv->tiling_mode != I915_TILING_NONE)
1463 i915_gem_object_save_bit_17_swizzle(obj);
1464
Chris Wilson3ef94da2009-09-14 16:50:29 +01001465 if (obj_priv->madv == I915_MADV_DONTNEED)
Chris Wilson13a05fd2009-09-20 23:03:19 +01001466 obj_priv->dirty = 0;
Chris Wilson3ef94da2009-09-14 16:50:29 +01001467
1468 for (i = 0; i < page_count; i++) {
1469 if (obj_priv->pages[i] == NULL)
1470 break;
1471
1472 if (obj_priv->dirty)
1473 set_page_dirty(obj_priv->pages[i]);
1474
1475 if (obj_priv->madv == I915_MADV_WILLNEED)
Eric Anholt856fa192009-03-19 14:10:50 -07001476 mark_page_accessed(obj_priv->pages[i]);
Chris Wilson3ef94da2009-09-14 16:50:29 +01001477
1478 page_cache_release(obj_priv->pages[i]);
1479 }
Eric Anholt673a3942008-07-30 12:06:12 -07001480 obj_priv->dirty = 0;
1481
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07001482 drm_free_large(obj_priv->pages);
Eric Anholt856fa192009-03-19 14:10:50 -07001483 obj_priv->pages = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07001484}
1485
1486static void
Eric Anholtce44b0e2008-11-06 16:00:31 -08001487i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
Eric Anholt673a3942008-07-30 12:06:12 -07001488{
1489 struct drm_device *dev = obj->dev;
1490 drm_i915_private_t *dev_priv = dev->dev_private;
1491 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1492
1493 /* Add a reference if we're newly entering the active list. */
1494 if (!obj_priv->active) {
1495 drm_gem_object_reference(obj);
1496 obj_priv->active = 1;
1497 }
1498 /* Move from whatever list we were on to the tail of execution. */
Carl Worth5e118f42009-03-20 11:54:25 -07001499 spin_lock(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07001500 list_move_tail(&obj_priv->list,
1501 &dev_priv->mm.active_list);
Carl Worth5e118f42009-03-20 11:54:25 -07001502 spin_unlock(&dev_priv->mm.active_list_lock);
Eric Anholtce44b0e2008-11-06 16:00:31 -08001503 obj_priv->last_rendering_seqno = seqno;
Eric Anholt673a3942008-07-30 12:06:12 -07001504}
1505
Eric Anholtce44b0e2008-11-06 16:00:31 -08001506static void
1507i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1508{
1509 struct drm_device *dev = obj->dev;
1510 drm_i915_private_t *dev_priv = dev->dev_private;
1511 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1512
1513 BUG_ON(!obj_priv->active);
1514 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1515 obj_priv->last_rendering_seqno = 0;
1516}
Eric Anholt673a3942008-07-30 12:06:12 -07001517
Chris Wilson963b4832009-09-20 23:03:54 +01001518/* Immediately discard the backing storage */
1519static void
1520i915_gem_object_truncate(struct drm_gem_object *obj)
1521{
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001522 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1523 struct inode *inode;
Chris Wilson963b4832009-09-20 23:03:54 +01001524
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001525 inode = obj->filp->f_path.dentry->d_inode;
1526 if (inode->i_op->truncate)
1527 inode->i_op->truncate (inode);
1528
1529 obj_priv->madv = __I915_MADV_PURGED;
Chris Wilson963b4832009-09-20 23:03:54 +01001530}
1531
1532static inline int
1533i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1534{
1535 return obj_priv->madv == I915_MADV_DONTNEED;
1536}
1537
Eric Anholt673a3942008-07-30 12:06:12 -07001538static void
1539i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1540{
1541 struct drm_device *dev = obj->dev;
1542 drm_i915_private_t *dev_priv = dev->dev_private;
1543 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1544
1545 i915_verify_inactive(dev, __FILE__, __LINE__);
1546 if (obj_priv->pin_count != 0)
1547 list_del_init(&obj_priv->list);
1548 else
1549 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1550
Daniel Vetter99fcb762010-02-07 16:20:18 +01001551 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1552
Eric Anholtce44b0e2008-11-06 16:00:31 -08001553 obj_priv->last_rendering_seqno = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07001554 if (obj_priv->active) {
1555 obj_priv->active = 0;
1556 drm_gem_object_unreference(obj);
1557 }
1558 i915_verify_inactive(dev, __FILE__, __LINE__);
1559}
1560
1561/**
1562 * Creates a new sequence number, emitting a write of it to the status page
1563 * plus an interrupt, which will trigger i915_user_interrupt_handler.
1564 *
1565 * Must be called with struct_lock held.
1566 *
1567 * Returned sequence numbers are nonzero on success.
1568 */
Daniel Vetter5a5a0c62009-09-15 22:57:36 +02001569uint32_t
Eric Anholtb9624422009-06-03 07:27:35 +00001570i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
1571 uint32_t flush_domains)
Eric Anholt673a3942008-07-30 12:06:12 -07001572{
1573 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholtb9624422009-06-03 07:27:35 +00001574 struct drm_i915_file_private *i915_file_priv = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07001575 struct drm_i915_gem_request *request;
1576 uint32_t seqno;
1577 int was_empty;
1578 RING_LOCALS;
1579
Eric Anholtb9624422009-06-03 07:27:35 +00001580 if (file_priv != NULL)
1581 i915_file_priv = file_priv->driver_priv;
1582
Eric Anholt9a298b22009-03-24 12:23:04 -07001583 request = kzalloc(sizeof(*request), GFP_KERNEL);
Eric Anholt673a3942008-07-30 12:06:12 -07001584 if (request == NULL)
1585 return 0;
1586
1587 /* Grab the seqno we're going to make this request be, and bump the
1588 * next (skipping 0 so it can be the reserved no-seqno value).
1589 */
1590 seqno = dev_priv->mm.next_gem_seqno;
1591 dev_priv->mm.next_gem_seqno++;
1592 if (dev_priv->mm.next_gem_seqno == 0)
1593 dev_priv->mm.next_gem_seqno++;
1594
1595 BEGIN_LP_RING(4);
1596 OUT_RING(MI_STORE_DWORD_INDEX);
1597 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
1598 OUT_RING(seqno);
1599
1600 OUT_RING(MI_USER_INTERRUPT);
1601 ADVANCE_LP_RING();
1602
Zhao Yakui44d98a62009-10-09 11:39:40 +08001603 DRM_DEBUG_DRIVER("%d\n", seqno);
Eric Anholt673a3942008-07-30 12:06:12 -07001604
1605 request->seqno = seqno;
1606 request->emitted_jiffies = jiffies;
Eric Anholt673a3942008-07-30 12:06:12 -07001607 was_empty = list_empty(&dev_priv->mm.request_list);
1608 list_add_tail(&request->list, &dev_priv->mm.request_list);
Eric Anholtb9624422009-06-03 07:27:35 +00001609 if (i915_file_priv) {
1610 list_add_tail(&request->client_list,
1611 &i915_file_priv->mm.request_list);
1612 } else {
1613 INIT_LIST_HEAD(&request->client_list);
1614 }
Eric Anholt673a3942008-07-30 12:06:12 -07001615
Eric Anholtce44b0e2008-11-06 16:00:31 -08001616 /* Associate any objects on the flushing list matching the write
1617 * domain we're flushing with our flush.
1618 */
1619 if (flush_domains != 0) {
1620 struct drm_i915_gem_object *obj_priv, *next;
1621
1622 list_for_each_entry_safe(obj_priv, next,
Daniel Vetter99fcb762010-02-07 16:20:18 +01001623 &dev_priv->mm.gpu_write_list,
1624 gpu_write_list) {
Eric Anholtce44b0e2008-11-06 16:00:31 -08001625 struct drm_gem_object *obj = obj_priv->obj;
1626
1627 if ((obj->write_domain & flush_domains) ==
1628 obj->write_domain) {
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001629 uint32_t old_write_domain = obj->write_domain;
1630
Eric Anholtce44b0e2008-11-06 16:00:31 -08001631 obj->write_domain = 0;
Daniel Vetter99fcb762010-02-07 16:20:18 +01001632 list_del_init(&obj_priv->gpu_write_list);
Eric Anholtce44b0e2008-11-06 16:00:31 -08001633 i915_gem_object_move_to_active(obj, seqno);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001634
1635 trace_i915_gem_object_change_domain(obj,
1636 obj->read_domains,
1637 old_write_domain);
Eric Anholtce44b0e2008-11-06 16:00:31 -08001638 }
1639 }
1640
1641 }
1642
Ben Gamarif65d9422009-09-14 17:48:44 -04001643 if (!dev_priv->mm.suspended) {
1644 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1645 if (was_empty)
1646 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1647 }
Eric Anholt673a3942008-07-30 12:06:12 -07001648 return seqno;
1649}
1650
1651/**
1652 * Command execution barrier
1653 *
1654 * Ensures that all commands in the ring are finished
1655 * before signalling the CPU
1656 */
Eric Anholt3043c602008-10-02 12:24:47 -07001657static uint32_t
Eric Anholt673a3942008-07-30 12:06:12 -07001658i915_retire_commands(struct drm_device *dev)
1659{
1660 drm_i915_private_t *dev_priv = dev->dev_private;
1661 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1662 uint32_t flush_domains = 0;
1663 RING_LOCALS;
1664
1665 /* The sampler always gets flushed on i965 (sigh) */
1666 if (IS_I965G(dev))
1667 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
1668 BEGIN_LP_RING(2);
1669 OUT_RING(cmd);
1670 OUT_RING(0); /* noop */
1671 ADVANCE_LP_RING();
1672 return flush_domains;
1673}
1674
1675/**
1676 * Moves buffers associated only with the given active seqno from the active
1677 * to inactive list, potentially freeing them.
1678 */
1679static void
1680i915_gem_retire_request(struct drm_device *dev,
1681 struct drm_i915_gem_request *request)
1682{
1683 drm_i915_private_t *dev_priv = dev->dev_private;
1684
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001685 trace_i915_gem_request_retire(dev, request->seqno);
1686
Eric Anholt673a3942008-07-30 12:06:12 -07001687 /* Move any buffers on the active list that are no longer referenced
1688 * by the ringbuffer to the flushing/inactive lists as appropriate.
1689 */
Carl Worth5e118f42009-03-20 11:54:25 -07001690 spin_lock(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07001691 while (!list_empty(&dev_priv->mm.active_list)) {
1692 struct drm_gem_object *obj;
1693 struct drm_i915_gem_object *obj_priv;
1694
1695 obj_priv = list_first_entry(&dev_priv->mm.active_list,
1696 struct drm_i915_gem_object,
1697 list);
1698 obj = obj_priv->obj;
1699
1700 /* If the seqno being retired doesn't match the oldest in the
1701 * list, then the oldest in the list must still be newer than
1702 * this seqno.
1703 */
1704 if (obj_priv->last_rendering_seqno != request->seqno)
Carl Worth5e118f42009-03-20 11:54:25 -07001705 goto out;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001706
Eric Anholt673a3942008-07-30 12:06:12 -07001707#if WATCH_LRU
1708 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1709 __func__, request->seqno, obj);
1710#endif
1711
Eric Anholtce44b0e2008-11-06 16:00:31 -08001712 if (obj->write_domain != 0)
1713 i915_gem_object_move_to_flushing(obj);
Shaohua Li68c84342009-04-08 10:58:23 +08001714 else {
1715 /* Take a reference on the object so it won't be
1716 * freed while the spinlock is held. The list
1717 * protection for this spinlock is safe when breaking
1718 * the lock like this since the next thing we do
1719 * is just get the head of the list again.
1720 */
1721 drm_gem_object_reference(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001722 i915_gem_object_move_to_inactive(obj);
Shaohua Li68c84342009-04-08 10:58:23 +08001723 spin_unlock(&dev_priv->mm.active_list_lock);
1724 drm_gem_object_unreference(obj);
1725 spin_lock(&dev_priv->mm.active_list_lock);
1726 }
Eric Anholt673a3942008-07-30 12:06:12 -07001727 }
Carl Worth5e118f42009-03-20 11:54:25 -07001728out:
1729 spin_unlock(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07001730}
1731
1732/**
1733 * Returns true if seq1 is later than seq2.
1734 */
Ben Gamari22be1722009-09-14 17:48:43 -04001735bool
Eric Anholt673a3942008-07-30 12:06:12 -07001736i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1737{
1738 return (int32_t)(seq1 - seq2) >= 0;
1739}
1740
1741uint32_t
1742i915_get_gem_seqno(struct drm_device *dev)
1743{
1744 drm_i915_private_t *dev_priv = dev->dev_private;
1745
1746 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
1747}
1748
1749/**
1750 * This function clears the request list as sequence numbers are passed.
1751 */
1752void
1753i915_gem_retire_requests(struct drm_device *dev)
1754{
1755 drm_i915_private_t *dev_priv = dev->dev_private;
1756 uint32_t seqno;
1757
Chris Wilson9d34e5d2009-09-24 05:26:06 +01001758 if (!dev_priv->hw_status_page || list_empty(&dev_priv->mm.request_list))
Karsten Wiese6c0594a2009-02-23 15:07:57 +01001759 return;
1760
Eric Anholt673a3942008-07-30 12:06:12 -07001761 seqno = i915_get_gem_seqno(dev);
1762
1763 while (!list_empty(&dev_priv->mm.request_list)) {
1764 struct drm_i915_gem_request *request;
1765 uint32_t retiring_seqno;
1766
1767 request = list_first_entry(&dev_priv->mm.request_list,
1768 struct drm_i915_gem_request,
1769 list);
1770 retiring_seqno = request->seqno;
1771
1772 if (i915_seqno_passed(seqno, retiring_seqno) ||
Ben Gamariba1234d2009-09-14 17:48:47 -04001773 atomic_read(&dev_priv->mm.wedged)) {
Eric Anholt673a3942008-07-30 12:06:12 -07001774 i915_gem_retire_request(dev, request);
1775
1776 list_del(&request->list);
Eric Anholtb9624422009-06-03 07:27:35 +00001777 list_del(&request->client_list);
Eric Anholt9a298b22009-03-24 12:23:04 -07001778 kfree(request);
Eric Anholt673a3942008-07-30 12:06:12 -07001779 } else
1780 break;
1781 }
Chris Wilson9d34e5d2009-09-24 05:26:06 +01001782
1783 if (unlikely (dev_priv->trace_irq_seqno &&
1784 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
1785 i915_user_irq_put(dev);
1786 dev_priv->trace_irq_seqno = 0;
1787 }
Eric Anholt673a3942008-07-30 12:06:12 -07001788}
1789
1790void
1791i915_gem_retire_work_handler(struct work_struct *work)
1792{
1793 drm_i915_private_t *dev_priv;
1794 struct drm_device *dev;
1795
1796 dev_priv = container_of(work, drm_i915_private_t,
1797 mm.retire_work.work);
1798 dev = dev_priv->dev;
1799
1800 mutex_lock(&dev->struct_mutex);
1801 i915_gem_retire_requests(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07001802 if (!dev_priv->mm.suspended &&
1803 !list_empty(&dev_priv->mm.request_list))
Eric Anholt9c9fe1f2009-08-03 16:09:16 -07001804 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
Eric Anholt673a3942008-07-30 12:06:12 -07001805 mutex_unlock(&dev->struct_mutex);
1806}
1807
Daniel Vetter5a5a0c62009-09-15 22:57:36 +02001808int
Daniel Vetter48764bf2009-09-15 22:57:32 +02001809i915_do_wait_request(struct drm_device *dev, uint32_t seqno, int interruptible)
Eric Anholt673a3942008-07-30 12:06:12 -07001810{
1811 drm_i915_private_t *dev_priv = dev->dev_private;
Jesse Barnes802c7eb2009-05-05 16:03:48 -07001812 u32 ier;
Eric Anholt673a3942008-07-30 12:06:12 -07001813 int ret = 0;
1814
1815 BUG_ON(seqno == 0);
1816
Ben Gamariba1234d2009-09-14 17:48:47 -04001817 if (atomic_read(&dev_priv->mm.wedged))
Ben Gamariffed1d02009-09-14 17:48:41 -04001818 return -EIO;
1819
Eric Anholt673a3942008-07-30 12:06:12 -07001820 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
Eric Anholtbad720f2009-10-22 16:11:14 -07001821 if (HAS_PCH_SPLIT(dev))
Zhenyu Wang036a4a72009-06-08 14:40:19 +08001822 ier = I915_READ(DEIER) | I915_READ(GTIER);
1823 else
1824 ier = I915_READ(IER);
Jesse Barnes802c7eb2009-05-05 16:03:48 -07001825 if (!ier) {
1826 DRM_ERROR("something (likely vbetool) disabled "
1827 "interrupts, re-enabling\n");
1828 i915_driver_irq_preinstall(dev);
1829 i915_driver_irq_postinstall(dev);
1830 }
1831
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001832 trace_i915_gem_request_wait_begin(dev, seqno);
1833
Eric Anholt673a3942008-07-30 12:06:12 -07001834 dev_priv->mm.waiting_gem_seqno = seqno;
1835 i915_user_irq_get(dev);
Daniel Vetter48764bf2009-09-15 22:57:32 +02001836 if (interruptible)
1837 ret = wait_event_interruptible(dev_priv->irq_queue,
1838 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1839 atomic_read(&dev_priv->mm.wedged));
1840 else
1841 wait_event(dev_priv->irq_queue,
1842 i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
1843 atomic_read(&dev_priv->mm.wedged));
1844
Eric Anholt673a3942008-07-30 12:06:12 -07001845 i915_user_irq_put(dev);
1846 dev_priv->mm.waiting_gem_seqno = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001847
1848 trace_i915_gem_request_wait_end(dev, seqno);
Eric Anholt673a3942008-07-30 12:06:12 -07001849 }
Ben Gamariba1234d2009-09-14 17:48:47 -04001850 if (atomic_read(&dev_priv->mm.wedged))
Eric Anholt673a3942008-07-30 12:06:12 -07001851 ret = -EIO;
1852
1853 if (ret && ret != -ERESTARTSYS)
1854 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
1855 __func__, ret, seqno, i915_get_gem_seqno(dev));
1856
1857 /* Directly dispatch request retiring. While we have the work queue
1858 * to handle this, the waiter on a request often wants an associated
1859 * buffer to have made it to the inactive list, and we would need
1860 * a separate wait queue to handle that.
1861 */
1862 if (ret == 0)
1863 i915_gem_retire_requests(dev);
1864
1865 return ret;
1866}
1867
Daniel Vetter48764bf2009-09-15 22:57:32 +02001868/**
1869 * Waits for a sequence number to be signaled, and cleans up the
1870 * request and object lists appropriately for that event.
1871 */
1872static int
1873i915_wait_request(struct drm_device *dev, uint32_t seqno)
1874{
1875 return i915_do_wait_request(dev, seqno, 1);
1876}
1877
Eric Anholt673a3942008-07-30 12:06:12 -07001878static void
1879i915_gem_flush(struct drm_device *dev,
1880 uint32_t invalidate_domains,
1881 uint32_t flush_domains)
1882{
1883 drm_i915_private_t *dev_priv = dev->dev_private;
1884 uint32_t cmd;
1885 RING_LOCALS;
1886
1887#if WATCH_EXEC
1888 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
1889 invalidate_domains, flush_domains);
1890#endif
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001891 trace_i915_gem_request_flush(dev, dev_priv->mm.next_gem_seqno,
1892 invalidate_domains, flush_domains);
Eric Anholt673a3942008-07-30 12:06:12 -07001893
1894 if (flush_domains & I915_GEM_DOMAIN_CPU)
1895 drm_agp_chipset_flush(dev);
1896
Chris Wilson21d509e2009-06-06 09:46:02 +01001897 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
Eric Anholt673a3942008-07-30 12:06:12 -07001898 /*
1899 * read/write caches:
1900 *
1901 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
1902 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
1903 * also flushed at 2d versus 3d pipeline switches.
1904 *
1905 * read-only caches:
1906 *
1907 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
1908 * MI_READ_FLUSH is set, and is always flushed on 965.
1909 *
1910 * I915_GEM_DOMAIN_COMMAND may not exist?
1911 *
1912 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
1913 * invalidated when MI_EXE_FLUSH is set.
1914 *
1915 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
1916 * invalidated with every MI_FLUSH.
1917 *
1918 * TLBs:
1919 *
1920 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
1921 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
1922 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
1923 * are flushed at any MI_FLUSH.
1924 */
1925
1926 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
1927 if ((invalidate_domains|flush_domains) &
1928 I915_GEM_DOMAIN_RENDER)
1929 cmd &= ~MI_NO_WRITE_FLUSH;
1930 if (!IS_I965G(dev)) {
1931 /*
1932 * On the 965, the sampler cache always gets flushed
1933 * and this bit is reserved.
1934 */
1935 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
1936 cmd |= MI_READ_FLUSH;
1937 }
1938 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
1939 cmd |= MI_EXE_FLUSH;
1940
1941#if WATCH_EXEC
1942 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
1943#endif
1944 BEGIN_LP_RING(2);
1945 OUT_RING(cmd);
Daniel Vetter48764bf2009-09-15 22:57:32 +02001946 OUT_RING(MI_NOOP);
Eric Anholt673a3942008-07-30 12:06:12 -07001947 ADVANCE_LP_RING();
1948 }
1949}
1950
1951/**
1952 * Ensures that all rendering to the object has completed and the object is
1953 * safe to unbind from the GTT or access from the CPU.
1954 */
1955static int
1956i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1957{
1958 struct drm_device *dev = obj->dev;
1959 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1960 int ret;
1961
Eric Anholte47c68e2008-11-14 13:35:19 -08001962 /* This function only exists to support waiting for existing rendering,
1963 * not for emitting required flushes.
Eric Anholt673a3942008-07-30 12:06:12 -07001964 */
Eric Anholte47c68e2008-11-14 13:35:19 -08001965 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
Eric Anholt673a3942008-07-30 12:06:12 -07001966
1967 /* If there is rendering queued on the buffer being evicted, wait for
1968 * it.
1969 */
1970 if (obj_priv->active) {
1971#if WATCH_BUF
1972 DRM_INFO("%s: object %p wait for seqno %08x\n",
1973 __func__, obj, obj_priv->last_rendering_seqno);
1974#endif
1975 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
1976 if (ret != 0)
1977 return ret;
1978 }
1979
1980 return 0;
1981}
1982
1983/**
1984 * Unbinds an object from the GTT aperture.
1985 */
Jesse Barnes0f973f22009-01-26 17:10:45 -08001986int
Eric Anholt673a3942008-07-30 12:06:12 -07001987i915_gem_object_unbind(struct drm_gem_object *obj)
1988{
1989 struct drm_device *dev = obj->dev;
1990 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1991 int ret = 0;
1992
1993#if WATCH_BUF
1994 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1995 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1996#endif
1997 if (obj_priv->gtt_space == NULL)
1998 return 0;
1999
2000 if (obj_priv->pin_count != 0) {
2001 DRM_ERROR("Attempting to unbind pinned buffer\n");
2002 return -EINVAL;
2003 }
2004
Eric Anholt5323fd02009-09-09 11:50:45 -07002005 /* blow away mappings if mapped through GTT */
2006 i915_gem_release_mmap(obj);
2007
Eric Anholt673a3942008-07-30 12:06:12 -07002008 /* Move the object to the CPU domain to ensure that
2009 * any possible CPU writes while it's not in the GTT
2010 * are flushed when we go to remap it. This will
2011 * also ensure that all pending GPU writes are finished
2012 * before we unbind.
2013 */
Eric Anholte47c68e2008-11-14 13:35:19 -08002014 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
Eric Anholt673a3942008-07-30 12:06:12 -07002015 if (ret) {
Eric Anholte47c68e2008-11-14 13:35:19 -08002016 if (ret != -ERESTARTSYS)
2017 DRM_ERROR("set_domain failed: %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07002018 return ret;
2019 }
2020
Eric Anholt5323fd02009-09-09 11:50:45 -07002021 BUG_ON(obj_priv->active);
2022
Daniel Vetter96b47b62009-12-15 17:50:00 +01002023 /* release the fence reg _after_ flushing */
2024 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2025 i915_gem_clear_fence_reg(obj);
2026
Eric Anholt673a3942008-07-30 12:06:12 -07002027 if (obj_priv->agp_mem != NULL) {
2028 drm_unbind_agp(obj_priv->agp_mem);
2029 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2030 obj_priv->agp_mem = NULL;
2031 }
2032
Eric Anholt856fa192009-03-19 14:10:50 -07002033 i915_gem_object_put_pages(obj);
Chris Wilsona32808c2009-09-20 21:29:47 +01002034 BUG_ON(obj_priv->pages_refcount);
Eric Anholt673a3942008-07-30 12:06:12 -07002035
2036 if (obj_priv->gtt_space) {
2037 atomic_dec(&dev->gtt_count);
2038 atomic_sub(obj->size, &dev->gtt_memory);
2039
2040 drm_mm_put_block(obj_priv->gtt_space);
2041 obj_priv->gtt_space = NULL;
2042 }
2043
2044 /* Remove ourselves from the LRU list if present. */
2045 if (!list_empty(&obj_priv->list))
2046 list_del_init(&obj_priv->list);
2047
Chris Wilson963b4832009-09-20 23:03:54 +01002048 if (i915_gem_object_is_purgeable(obj_priv))
2049 i915_gem_object_truncate(obj);
2050
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002051 trace_i915_gem_object_unbind(obj);
2052
Eric Anholt673a3942008-07-30 12:06:12 -07002053 return 0;
2054}
2055
Chris Wilson07f73f62009-09-14 16:50:30 +01002056static struct drm_gem_object *
2057i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
2058{
2059 drm_i915_private_t *dev_priv = dev->dev_private;
2060 struct drm_i915_gem_object *obj_priv;
2061 struct drm_gem_object *best = NULL;
2062 struct drm_gem_object *first = NULL;
2063
2064 /* Try to find the smallest clean object */
2065 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
2066 struct drm_gem_object *obj = obj_priv->obj;
2067 if (obj->size >= min_size) {
Chris Wilson963b4832009-09-20 23:03:54 +01002068 if ((!obj_priv->dirty ||
2069 i915_gem_object_is_purgeable(obj_priv)) &&
Chris Wilson07f73f62009-09-14 16:50:30 +01002070 (!best || obj->size < best->size)) {
2071 best = obj;
2072 if (best->size == min_size)
2073 return best;
2074 }
2075 if (!first)
2076 first = obj;
2077 }
2078 }
2079
2080 return best ? best : first;
2081}
2082
Eric Anholt673a3942008-07-30 12:06:12 -07002083static int
Chris Wilson07f73f62009-09-14 16:50:30 +01002084i915_gem_evict_everything(struct drm_device *dev)
2085{
2086 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson07f73f62009-09-14 16:50:30 +01002087 int ret;
Daniel Vetter99fcb762010-02-07 16:20:18 +01002088 uint32_t seqno;
Chris Wilson07f73f62009-09-14 16:50:30 +01002089 bool lists_empty;
2090
Chris Wilson07f73f62009-09-14 16:50:30 +01002091 spin_lock(&dev_priv->mm.active_list_lock);
2092 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2093 list_empty(&dev_priv->mm.flushing_list) &&
2094 list_empty(&dev_priv->mm.active_list));
2095 spin_unlock(&dev_priv->mm.active_list_lock);
2096
Chris Wilson97311292009-09-21 00:22:34 +01002097 if (lists_empty)
Chris Wilson07f73f62009-09-14 16:50:30 +01002098 return -ENOSPC;
Chris Wilson07f73f62009-09-14 16:50:30 +01002099
2100 /* Flush everything (on to the inactive lists) and evict */
2101 i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2102 seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
2103 if (seqno == 0)
2104 return -ENOMEM;
2105
2106 ret = i915_wait_request(dev, seqno);
2107 if (ret)
2108 return ret;
2109
Daniel Vetter99fcb762010-02-07 16:20:18 +01002110 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2111
Chris Wilsonab5ee572009-09-20 19:25:47 +01002112 ret = i915_gem_evict_from_inactive_list(dev);
Chris Wilson07f73f62009-09-14 16:50:30 +01002113 if (ret)
2114 return ret;
2115
2116 spin_lock(&dev_priv->mm.active_list_lock);
2117 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2118 list_empty(&dev_priv->mm.flushing_list) &&
2119 list_empty(&dev_priv->mm.active_list));
2120 spin_unlock(&dev_priv->mm.active_list_lock);
2121 BUG_ON(!lists_empty);
2122
Eric Anholt673a3942008-07-30 12:06:12 -07002123 return 0;
2124}
2125
2126static int
Chris Wilson07f73f62009-09-14 16:50:30 +01002127i915_gem_evict_something(struct drm_device *dev, int min_size)
Eric Anholt673a3942008-07-30 12:06:12 -07002128{
2129 drm_i915_private_t *dev_priv = dev->dev_private;
2130 struct drm_gem_object *obj;
Chris Wilson07f73f62009-09-14 16:50:30 +01002131 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002132
2133 for (;;) {
Chris Wilson07f73f62009-09-14 16:50:30 +01002134 i915_gem_retire_requests(dev);
2135
Eric Anholt673a3942008-07-30 12:06:12 -07002136 /* If there's an inactive buffer available now, grab it
2137 * and be done.
2138 */
Chris Wilson07f73f62009-09-14 16:50:30 +01002139 obj = i915_gem_find_inactive_object(dev, min_size);
2140 if (obj) {
2141 struct drm_i915_gem_object *obj_priv;
2142
Eric Anholt673a3942008-07-30 12:06:12 -07002143#if WATCH_LRU
2144 DRM_INFO("%s: evicting %p\n", __func__, obj);
2145#endif
Chris Wilson07f73f62009-09-14 16:50:30 +01002146 obj_priv = obj->driver_private;
2147 BUG_ON(obj_priv->pin_count != 0);
Eric Anholt673a3942008-07-30 12:06:12 -07002148 BUG_ON(obj_priv->active);
2149
2150 /* Wait on the rendering and unbind the buffer. */
Chris Wilson07f73f62009-09-14 16:50:30 +01002151 return i915_gem_object_unbind(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002152 }
2153
2154 /* If we didn't get anything, but the ring is still processing
Chris Wilson07f73f62009-09-14 16:50:30 +01002155 * things, wait for the next to finish and hopefully leave us
2156 * a buffer to evict.
Eric Anholt673a3942008-07-30 12:06:12 -07002157 */
2158 if (!list_empty(&dev_priv->mm.request_list)) {
2159 struct drm_i915_gem_request *request;
2160
2161 request = list_first_entry(&dev_priv->mm.request_list,
2162 struct drm_i915_gem_request,
2163 list);
2164
2165 ret = i915_wait_request(dev, request->seqno);
2166 if (ret)
Chris Wilson07f73f62009-09-14 16:50:30 +01002167 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002168
Chris Wilson07f73f62009-09-14 16:50:30 +01002169 continue;
Eric Anholt673a3942008-07-30 12:06:12 -07002170 }
2171
2172 /* If we didn't have anything on the request list but there
2173 * are buffers awaiting a flush, emit one and try again.
2174 * When we wait on it, those buffers waiting for that flush
2175 * will get moved to inactive.
2176 */
2177 if (!list_empty(&dev_priv->mm.flushing_list)) {
Chris Wilson07f73f62009-09-14 16:50:30 +01002178 struct drm_i915_gem_object *obj_priv;
Eric Anholt673a3942008-07-30 12:06:12 -07002179
Chris Wilson9a1e2582009-09-20 20:16:50 +01002180 /* Find an object that we can immediately reuse */
2181 list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
2182 obj = obj_priv->obj;
2183 if (obj->size >= min_size)
2184 break;
Eric Anholt673a3942008-07-30 12:06:12 -07002185
Chris Wilson9a1e2582009-09-20 20:16:50 +01002186 obj = NULL;
2187 }
Eric Anholt673a3942008-07-30 12:06:12 -07002188
Chris Wilson9a1e2582009-09-20 20:16:50 +01002189 if (obj != NULL) {
2190 uint32_t seqno;
Chris Wilson07f73f62009-09-14 16:50:30 +01002191
Chris Wilson9a1e2582009-09-20 20:16:50 +01002192 i915_gem_flush(dev,
2193 obj->write_domain,
2194 obj->write_domain);
2195 seqno = i915_add_request(dev, NULL, obj->write_domain);
2196 if (seqno == 0)
2197 return -ENOMEM;
2198
2199 ret = i915_wait_request(dev, seqno);
2200 if (ret)
2201 return ret;
2202
2203 continue;
2204 }
Eric Anholt673a3942008-07-30 12:06:12 -07002205 }
2206
Chris Wilson07f73f62009-09-14 16:50:30 +01002207 /* If we didn't do any of the above, there's no single buffer
2208 * large enough to swap out for the new one, so just evict
2209 * everything and start again. (This should be rare.)
Eric Anholt673a3942008-07-30 12:06:12 -07002210 */
Chris Wilson97311292009-09-21 00:22:34 +01002211 if (!list_empty (&dev_priv->mm.inactive_list))
Chris Wilsonab5ee572009-09-20 19:25:47 +01002212 return i915_gem_evict_from_inactive_list(dev);
Chris Wilson97311292009-09-21 00:22:34 +01002213 else
Chris Wilson07f73f62009-09-14 16:50:30 +01002214 return i915_gem_evict_everything(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07002215 }
Keith Packardac94a962008-11-20 23:30:27 -08002216}
2217
Ben Gamari6911a9b2009-04-02 11:24:54 -07002218int
Chris Wilson4bdadb92010-01-27 13:36:32 +00002219i915_gem_object_get_pages(struct drm_gem_object *obj,
2220 gfp_t gfpmask)
Eric Anholt673a3942008-07-30 12:06:12 -07002221{
2222 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2223 int page_count, i;
2224 struct address_space *mapping;
2225 struct inode *inode;
2226 struct page *page;
2227 int ret;
2228
Eric Anholt856fa192009-03-19 14:10:50 -07002229 if (obj_priv->pages_refcount++ != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07002230 return 0;
2231
2232 /* Get the list of pages out of our struct file. They'll be pinned
2233 * at this point until we release them.
2234 */
2235 page_count = obj->size / PAGE_SIZE;
Eric Anholt856fa192009-03-19 14:10:50 -07002236 BUG_ON(obj_priv->pages != NULL);
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07002237 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
Eric Anholt856fa192009-03-19 14:10:50 -07002238 if (obj_priv->pages == NULL) {
Eric Anholt856fa192009-03-19 14:10:50 -07002239 obj_priv->pages_refcount--;
Eric Anholt673a3942008-07-30 12:06:12 -07002240 return -ENOMEM;
2241 }
2242
2243 inode = obj->filp->f_path.dentry->d_inode;
2244 mapping = inode->i_mapping;
2245 for (i = 0; i < page_count; i++) {
Chris Wilson4bdadb92010-01-27 13:36:32 +00002246 page = read_cache_page_gfp(mapping, i,
2247 mapping_gfp_mask (mapping) |
2248 __GFP_COLD |
2249 gfpmask);
Eric Anholt673a3942008-07-30 12:06:12 -07002250 if (IS_ERR(page)) {
2251 ret = PTR_ERR(page);
Eric Anholt856fa192009-03-19 14:10:50 -07002252 i915_gem_object_put_pages(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002253 return ret;
2254 }
Eric Anholt856fa192009-03-19 14:10:50 -07002255 obj_priv->pages[i] = page;
Eric Anholt673a3942008-07-30 12:06:12 -07002256 }
Eric Anholt280b7132009-03-12 16:56:27 -07002257
2258 if (obj_priv->tiling_mode != I915_TILING_NONE)
2259 i915_gem_object_do_bit_17_swizzle(obj);
2260
Eric Anholt673a3942008-07-30 12:06:12 -07002261 return 0;
2262}
2263
Eric Anholt4e901fd2009-10-26 16:44:17 -07002264static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2265{
2266 struct drm_gem_object *obj = reg->obj;
2267 struct drm_device *dev = obj->dev;
2268 drm_i915_private_t *dev_priv = dev->dev_private;
2269 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2270 int regnum = obj_priv->fence_reg;
2271 uint64_t val;
2272
2273 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2274 0xfffff000) << 32;
2275 val |= obj_priv->gtt_offset & 0xfffff000;
2276 val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2277 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2278
2279 if (obj_priv->tiling_mode == I915_TILING_Y)
2280 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2281 val |= I965_FENCE_REG_VALID;
2282
2283 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2284}
2285
Jesse Barnesde151cf2008-11-12 10:03:55 -08002286static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2287{
2288 struct drm_gem_object *obj = reg->obj;
2289 struct drm_device *dev = obj->dev;
2290 drm_i915_private_t *dev_priv = dev->dev_private;
2291 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2292 int regnum = obj_priv->fence_reg;
2293 uint64_t val;
2294
2295 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2296 0xfffff000) << 32;
2297 val |= obj_priv->gtt_offset & 0xfffff000;
2298 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2299 if (obj_priv->tiling_mode == I915_TILING_Y)
2300 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2301 val |= I965_FENCE_REG_VALID;
2302
2303 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2304}
2305
2306static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2307{
2308 struct drm_gem_object *obj = reg->obj;
2309 struct drm_device *dev = obj->dev;
2310 drm_i915_private_t *dev_priv = dev->dev_private;
2311 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2312 int regnum = obj_priv->fence_reg;
Jesse Barnes0f973f22009-01-26 17:10:45 -08002313 int tile_width;
Eric Anholtdc529a42009-03-10 22:34:49 -07002314 uint32_t fence_reg, val;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002315 uint32_t pitch_val;
2316
2317 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2318 (obj_priv->gtt_offset & (obj->size - 1))) {
Linus Torvaldsf06da262009-02-09 08:57:29 -08002319 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
Jesse Barnes0f973f22009-01-26 17:10:45 -08002320 __func__, obj_priv->gtt_offset, obj->size);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002321 return;
2322 }
2323
Jesse Barnes0f973f22009-01-26 17:10:45 -08002324 if (obj_priv->tiling_mode == I915_TILING_Y &&
2325 HAS_128_BYTE_Y_TILING(dev))
2326 tile_width = 128;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002327 else
Jesse Barnes0f973f22009-01-26 17:10:45 -08002328 tile_width = 512;
2329
2330 /* Note: pitch better be a power of two tile widths */
2331 pitch_val = obj_priv->stride / tile_width;
2332 pitch_val = ffs(pitch_val) - 1;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002333
2334 val = obj_priv->gtt_offset;
2335 if (obj_priv->tiling_mode == I915_TILING_Y)
2336 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2337 val |= I915_FENCE_SIZE_BITS(obj->size);
2338 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2339 val |= I830_FENCE_REG_VALID;
2340
Eric Anholtdc529a42009-03-10 22:34:49 -07002341 if (regnum < 8)
2342 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2343 else
2344 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2345 I915_WRITE(fence_reg, val);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002346}
2347
2348static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2349{
2350 struct drm_gem_object *obj = reg->obj;
2351 struct drm_device *dev = obj->dev;
2352 drm_i915_private_t *dev_priv = dev->dev_private;
2353 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2354 int regnum = obj_priv->fence_reg;
2355 uint32_t val;
2356 uint32_t pitch_val;
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002357 uint32_t fence_size_bits;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002358
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002359 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
Jesse Barnesde151cf2008-11-12 10:03:55 -08002360 (obj_priv->gtt_offset & (obj->size - 1))) {
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002361 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
Jesse Barnes0f973f22009-01-26 17:10:45 -08002362 __func__, obj_priv->gtt_offset);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002363 return;
2364 }
2365
Eric Anholte76a16d2009-05-26 17:44:56 -07002366 pitch_val = obj_priv->stride / 128;
2367 pitch_val = ffs(pitch_val) - 1;
2368 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2369
Jesse Barnesde151cf2008-11-12 10:03:55 -08002370 val = obj_priv->gtt_offset;
2371 if (obj_priv->tiling_mode == I915_TILING_Y)
2372 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002373 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2374 WARN_ON(fence_size_bits & ~0x00000f00);
2375 val |= fence_size_bits;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002376 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2377 val |= I830_FENCE_REG_VALID;
2378
2379 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002380}
2381
2382/**
2383 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2384 * @obj: object to map through a fence reg
2385 *
2386 * When mapping objects through the GTT, userspace wants to be able to write
2387 * to them without having to worry about swizzling if the object is tiled.
2388 *
2389 * This function walks the fence regs looking for a free one for @obj,
2390 * stealing one if it can't find any.
2391 *
2392 * It then sets up the reg based on the object's properties: address, pitch
2393 * and tiling format.
2394 */
Chris Wilson8c4b8c32009-06-17 22:08:52 +01002395int
2396i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002397{
2398 struct drm_device *dev = obj->dev;
Jesse Barnes79e53942008-11-07 14:24:08 -08002399 struct drm_i915_private *dev_priv = dev->dev_private;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002400 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2401 struct drm_i915_fence_reg *reg = NULL;
Chris Wilsonfc7170b2009-02-11 14:26:46 +00002402 struct drm_i915_gem_object *old_obj_priv = NULL;
2403 int i, ret, avail;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002404
Eric Anholta09ba7f2009-08-29 12:49:51 -07002405 /* Just update our place in the LRU if our fence is getting used. */
2406 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
2407 list_move_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2408 return 0;
2409 }
2410
Jesse Barnesde151cf2008-11-12 10:03:55 -08002411 switch (obj_priv->tiling_mode) {
2412 case I915_TILING_NONE:
2413 WARN(1, "allocating a fence for non-tiled object?\n");
2414 break;
2415 case I915_TILING_X:
Jesse Barnes0f973f22009-01-26 17:10:45 -08002416 if (!obj_priv->stride)
2417 return -EINVAL;
2418 WARN((obj_priv->stride & (512 - 1)),
2419 "object 0x%08x is X tiled but has non-512B pitch\n",
2420 obj_priv->gtt_offset);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002421 break;
2422 case I915_TILING_Y:
Jesse Barnes0f973f22009-01-26 17:10:45 -08002423 if (!obj_priv->stride)
2424 return -EINVAL;
2425 WARN((obj_priv->stride & (128 - 1)),
2426 "object 0x%08x is Y tiled but has non-128B pitch\n",
2427 obj_priv->gtt_offset);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002428 break;
2429 }
2430
2431 /* First try to find a free reg */
Chris Wilsonfc7170b2009-02-11 14:26:46 +00002432 avail = 0;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002433 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2434 reg = &dev_priv->fence_regs[i];
2435 if (!reg->obj)
2436 break;
Chris Wilsonfc7170b2009-02-11 14:26:46 +00002437
2438 old_obj_priv = reg->obj->driver_private;
2439 if (!old_obj_priv->pin_count)
2440 avail++;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002441 }
2442
2443 /* None available, try to steal one or wait for a user to finish */
2444 if (i == dev_priv->num_fence_regs) {
Eric Anholta09ba7f2009-08-29 12:49:51 -07002445 struct drm_gem_object *old_obj = NULL;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002446
Chris Wilsonfc7170b2009-02-11 14:26:46 +00002447 if (avail == 0)
Chris Wilson2939e1f2009-06-06 09:46:03 +01002448 return -ENOSPC;
Chris Wilsonfc7170b2009-02-11 14:26:46 +00002449
Eric Anholta09ba7f2009-08-29 12:49:51 -07002450 list_for_each_entry(old_obj_priv, &dev_priv->mm.fence_list,
2451 fence_list) {
2452 old_obj = old_obj_priv->obj;
Chris Wilsond7619c42009-02-11 14:26:47 +00002453
Chris Wilsond7619c42009-02-11 14:26:47 +00002454 if (old_obj_priv->pin_count)
2455 continue;
2456
Eric Anholta09ba7f2009-08-29 12:49:51 -07002457 /* Take a reference, as otherwise the wait_rendering
2458 * below may cause the object to get freed out from
2459 * under us.
2460 */
2461 drm_gem_object_reference(old_obj);
2462
Chris Wilsond7619c42009-02-11 14:26:47 +00002463 /* i915 uses fences for GPU access to tiled buffers */
2464 if (IS_I965G(dev) || !old_obj_priv->active)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002465 break;
Chris Wilsond7619c42009-02-11 14:26:47 +00002466
Eric Anholta09ba7f2009-08-29 12:49:51 -07002467 /* This brings the object to the head of the LRU if it
2468 * had been written to. The only way this should
2469 * result in us waiting longer than the expected
2470 * optimal amount of time is if there was a
2471 * fence-using buffer later that was read-only.
2472 */
2473 i915_gem_object_flush_gpu_write_domain(old_obj);
2474 ret = i915_gem_object_wait_rendering(old_obj);
Chris Wilson58c2fb62009-09-01 12:02:39 +01002475 if (ret != 0) {
2476 drm_gem_object_unreference(old_obj);
Chris Wilsond7619c42009-02-11 14:26:47 +00002477 return ret;
Chris Wilson58c2fb62009-09-01 12:02:39 +01002478 }
2479
Eric Anholta09ba7f2009-08-29 12:49:51 -07002480 break;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002481 }
2482
2483 /*
2484 * Zap this virtual mapping so we can set up a fence again
2485 * for this object next time we need it.
2486 */
Chris Wilson58c2fb62009-09-01 12:02:39 +01002487 i915_gem_release_mmap(old_obj);
2488
Eric Anholta09ba7f2009-08-29 12:49:51 -07002489 i = old_obj_priv->fence_reg;
Chris Wilson58c2fb62009-09-01 12:02:39 +01002490 reg = &dev_priv->fence_regs[i];
2491
Jesse Barnesde151cf2008-11-12 10:03:55 -08002492 old_obj_priv->fence_reg = I915_FENCE_REG_NONE;
Eric Anholta09ba7f2009-08-29 12:49:51 -07002493 list_del_init(&old_obj_priv->fence_list);
Chris Wilson58c2fb62009-09-01 12:02:39 +01002494
Eric Anholta09ba7f2009-08-29 12:49:51 -07002495 drm_gem_object_unreference(old_obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002496 }
2497
2498 obj_priv->fence_reg = i;
Eric Anholta09ba7f2009-08-29 12:49:51 -07002499 list_add_tail(&obj_priv->fence_list, &dev_priv->mm.fence_list);
2500
Jesse Barnesde151cf2008-11-12 10:03:55 -08002501 reg->obj = obj;
2502
Eric Anholt4e901fd2009-10-26 16:44:17 -07002503 if (IS_GEN6(dev))
2504 sandybridge_write_fence_reg(reg);
2505 else if (IS_I965G(dev))
Jesse Barnesde151cf2008-11-12 10:03:55 -08002506 i965_write_fence_reg(reg);
2507 else if (IS_I9XX(dev))
2508 i915_write_fence_reg(reg);
2509 else
2510 i830_write_fence_reg(reg);
Eric Anholtd9ddcb92009-01-27 10:33:49 -08002511
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002512 trace_i915_gem_object_get_fence(obj, i, obj_priv->tiling_mode);
2513
Eric Anholtd9ddcb92009-01-27 10:33:49 -08002514 return 0;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002515}
2516
2517/**
2518 * i915_gem_clear_fence_reg - clear out fence register info
2519 * @obj: object to clear
2520 *
2521 * Zeroes out the fence register itself and clears out the associated
2522 * data structures in dev_priv and obj_priv.
2523 */
2524static void
2525i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2526{
2527 struct drm_device *dev = obj->dev;
Jesse Barnes79e53942008-11-07 14:24:08 -08002528 drm_i915_private_t *dev_priv = dev->dev_private;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002529 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2530
Eric Anholt4e901fd2009-10-26 16:44:17 -07002531 if (IS_GEN6(dev)) {
2532 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2533 (obj_priv->fence_reg * 8), 0);
2534 } else if (IS_I965G(dev)) {
Jesse Barnesde151cf2008-11-12 10:03:55 -08002535 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
Eric Anholt4e901fd2009-10-26 16:44:17 -07002536 } else {
Eric Anholtdc529a42009-03-10 22:34:49 -07002537 uint32_t fence_reg;
2538
2539 if (obj_priv->fence_reg < 8)
2540 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2541 else
2542 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2543 8) * 4;
2544
2545 I915_WRITE(fence_reg, 0);
2546 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08002547
2548 dev_priv->fence_regs[obj_priv->fence_reg].obj = NULL;
2549 obj_priv->fence_reg = I915_FENCE_REG_NONE;
Eric Anholta09ba7f2009-08-29 12:49:51 -07002550 list_del_init(&obj_priv->fence_list);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002551}
2552
Eric Anholt673a3942008-07-30 12:06:12 -07002553/**
Chris Wilson52dc7d32009-06-06 09:46:01 +01002554 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2555 * to the buffer to finish, and then resets the fence register.
2556 * @obj: tiled object holding a fence register.
2557 *
2558 * Zeroes out the fence register itself and clears out the associated
2559 * data structures in dev_priv and obj_priv.
2560 */
2561int
2562i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2563{
2564 struct drm_device *dev = obj->dev;
2565 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2566
2567 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2568 return 0;
2569
Daniel Vetter10ae9bd2010-02-01 13:59:17 +01002570 /* If we've changed tiling, GTT-mappings of the object
2571 * need to re-fault to ensure that the correct fence register
2572 * setup is in place.
2573 */
2574 i915_gem_release_mmap(obj);
2575
Chris Wilson52dc7d32009-06-06 09:46:01 +01002576 /* On the i915, GPU access to tiled buffers is via a fence,
2577 * therefore we must wait for any outstanding access to complete
2578 * before clearing the fence.
2579 */
2580 if (!IS_I965G(dev)) {
2581 int ret;
2582
2583 i915_gem_object_flush_gpu_write_domain(obj);
Chris Wilson52dc7d32009-06-06 09:46:01 +01002584 ret = i915_gem_object_wait_rendering(obj);
2585 if (ret != 0)
2586 return ret;
2587 }
2588
Daniel Vetter4a726612010-02-01 13:59:16 +01002589 i915_gem_object_flush_gtt_write_domain(obj);
Chris Wilson52dc7d32009-06-06 09:46:01 +01002590 i915_gem_clear_fence_reg (obj);
2591
2592 return 0;
2593}
2594
2595/**
Eric Anholt673a3942008-07-30 12:06:12 -07002596 * Finds free space in the GTT aperture and binds the object there.
2597 */
2598static int
2599i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2600{
2601 struct drm_device *dev = obj->dev;
2602 drm_i915_private_t *dev_priv = dev->dev_private;
2603 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2604 struct drm_mm_node *free_space;
Chris Wilson4bdadb92010-01-27 13:36:32 +00002605 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
Chris Wilson07f73f62009-09-14 16:50:30 +01002606 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002607
Chris Wilsonbb6baf72009-09-22 14:24:13 +01002608 if (obj_priv->madv != I915_MADV_WILLNEED) {
Chris Wilson3ef94da2009-09-14 16:50:29 +01002609 DRM_ERROR("Attempting to bind a purgeable object\n");
2610 return -EINVAL;
2611 }
2612
Eric Anholt673a3942008-07-30 12:06:12 -07002613 if (alignment == 0)
Jesse Barnes0f973f22009-01-26 17:10:45 -08002614 alignment = i915_gem_get_gtt_alignment(obj);
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002615 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
Eric Anholt673a3942008-07-30 12:06:12 -07002616 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2617 return -EINVAL;
2618 }
2619
2620 search_free:
2621 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2622 obj->size, alignment, 0);
2623 if (free_space != NULL) {
2624 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2625 alignment);
2626 if (obj_priv->gtt_space != NULL) {
2627 obj_priv->gtt_space->private = obj;
2628 obj_priv->gtt_offset = obj_priv->gtt_space->start;
2629 }
2630 }
2631 if (obj_priv->gtt_space == NULL) {
2632 /* If the gtt is empty and we're still having trouble
2633 * fitting our object in, we're out of memory.
2634 */
2635#if WATCH_LRU
2636 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2637#endif
Chris Wilson07f73f62009-09-14 16:50:30 +01002638 ret = i915_gem_evict_something(dev, obj->size);
Chris Wilson97311292009-09-21 00:22:34 +01002639 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07002640 return ret;
Chris Wilson97311292009-09-21 00:22:34 +01002641
Eric Anholt673a3942008-07-30 12:06:12 -07002642 goto search_free;
2643 }
2644
2645#if WATCH_BUF
Krzysztof Halasacfd43c02009-06-20 00:31:28 +02002646 DRM_INFO("Binding object of size %zd at 0x%08x\n",
Eric Anholt673a3942008-07-30 12:06:12 -07002647 obj->size, obj_priv->gtt_offset);
2648#endif
Chris Wilson4bdadb92010-01-27 13:36:32 +00002649 ret = i915_gem_object_get_pages(obj, gfpmask);
Eric Anholt673a3942008-07-30 12:06:12 -07002650 if (ret) {
2651 drm_mm_put_block(obj_priv->gtt_space);
2652 obj_priv->gtt_space = NULL;
Chris Wilson07f73f62009-09-14 16:50:30 +01002653
2654 if (ret == -ENOMEM) {
2655 /* first try to clear up some space from the GTT */
2656 ret = i915_gem_evict_something(dev, obj->size);
2657 if (ret) {
Chris Wilson07f73f62009-09-14 16:50:30 +01002658 /* now try to shrink everyone else */
Chris Wilson4bdadb92010-01-27 13:36:32 +00002659 if (gfpmask) {
2660 gfpmask = 0;
2661 goto search_free;
Chris Wilson07f73f62009-09-14 16:50:30 +01002662 }
2663
2664 return ret;
2665 }
2666
2667 goto search_free;
2668 }
2669
Eric Anholt673a3942008-07-30 12:06:12 -07002670 return ret;
2671 }
2672
Eric Anholt673a3942008-07-30 12:06:12 -07002673 /* Create an AGP memory structure pointing at our pages, and bind it
2674 * into the GTT.
2675 */
2676 obj_priv->agp_mem = drm_agp_bind_pages(dev,
Eric Anholt856fa192009-03-19 14:10:50 -07002677 obj_priv->pages,
Chris Wilson07f73f62009-09-14 16:50:30 +01002678 obj->size >> PAGE_SHIFT,
Keith Packardba1eb1d2008-10-14 19:55:10 -07002679 obj_priv->gtt_offset,
2680 obj_priv->agp_type);
Eric Anholt673a3942008-07-30 12:06:12 -07002681 if (obj_priv->agp_mem == NULL) {
Eric Anholt856fa192009-03-19 14:10:50 -07002682 i915_gem_object_put_pages(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002683 drm_mm_put_block(obj_priv->gtt_space);
2684 obj_priv->gtt_space = NULL;
Chris Wilson07f73f62009-09-14 16:50:30 +01002685
2686 ret = i915_gem_evict_something(dev, obj->size);
Chris Wilson97311292009-09-21 00:22:34 +01002687 if (ret)
Chris Wilson07f73f62009-09-14 16:50:30 +01002688 return ret;
Chris Wilson07f73f62009-09-14 16:50:30 +01002689
2690 goto search_free;
Eric Anholt673a3942008-07-30 12:06:12 -07002691 }
2692 atomic_inc(&dev->gtt_count);
2693 atomic_add(obj->size, &dev->gtt_memory);
2694
2695 /* Assert that the object is not currently in any GPU domain. As it
2696 * wasn't in the GTT, there shouldn't be any way it could have been in
2697 * a GPU cache
2698 */
Chris Wilson21d509e2009-06-06 09:46:02 +01002699 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2700 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
Eric Anholt673a3942008-07-30 12:06:12 -07002701
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002702 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2703
Eric Anholt673a3942008-07-30 12:06:12 -07002704 return 0;
2705}
2706
2707void
2708i915_gem_clflush_object(struct drm_gem_object *obj)
2709{
2710 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2711
2712 /* If we don't have a page list set up, then we're not pinned
2713 * to GPU, and we can ignore the cache flush because it'll happen
2714 * again at bind time.
2715 */
Eric Anholt856fa192009-03-19 14:10:50 -07002716 if (obj_priv->pages == NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07002717 return;
2718
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002719 trace_i915_gem_object_clflush(obj);
Eric Anholtcfa16a02009-05-26 18:46:16 -07002720
Eric Anholt856fa192009-03-19 14:10:50 -07002721 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
Eric Anholt673a3942008-07-30 12:06:12 -07002722}
2723
Eric Anholte47c68e2008-11-14 13:35:19 -08002724/** Flushes any GPU write domain for the object if it's dirty. */
2725static void
2726i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2727{
2728 struct drm_device *dev = obj->dev;
2729 uint32_t seqno;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002730 uint32_t old_write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002731
2732 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2733 return;
2734
2735 /* Queue the GPU write cache flushing we need. */
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002736 old_write_domain = obj->write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002737 i915_gem_flush(dev, 0, obj->write_domain);
Eric Anholtb9624422009-06-03 07:27:35 +00002738 seqno = i915_add_request(dev, NULL, obj->write_domain);
Daniel Vetter99fcb762010-02-07 16:20:18 +01002739 BUG_ON(obj->write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08002740 i915_gem_object_move_to_active(obj, seqno);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002741
2742 trace_i915_gem_object_change_domain(obj,
2743 obj->read_domains,
2744 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08002745}
2746
2747/** Flushes the GTT write domain for the object if it's dirty. */
2748static void
2749i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2750{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002751 uint32_t old_write_domain;
2752
Eric Anholte47c68e2008-11-14 13:35:19 -08002753 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2754 return;
2755
2756 /* No actual flushing is required for the GTT write domain. Writes
2757 * to it immediately go to main memory as far as we know, so there's
2758 * no chipset flush. It also doesn't land in render cache.
2759 */
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002760 old_write_domain = obj->write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002761 obj->write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002762
2763 trace_i915_gem_object_change_domain(obj,
2764 obj->read_domains,
2765 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08002766}
2767
2768/** Flushes the CPU write domain for the object if it's dirty. */
2769static void
2770i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2771{
2772 struct drm_device *dev = obj->dev;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002773 uint32_t old_write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002774
2775 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2776 return;
2777
2778 i915_gem_clflush_object(obj);
2779 drm_agp_chipset_flush(dev);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002780 old_write_domain = obj->write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002781 obj->write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002782
2783 trace_i915_gem_object_change_domain(obj,
2784 obj->read_domains,
2785 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08002786}
2787
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05002788void
2789i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2790{
2791 switch (obj->write_domain) {
2792 case I915_GEM_DOMAIN_GTT:
2793 i915_gem_object_flush_gtt_write_domain(obj);
2794 break;
2795 case I915_GEM_DOMAIN_CPU:
2796 i915_gem_object_flush_cpu_write_domain(obj);
2797 break;
2798 default:
2799 i915_gem_object_flush_gpu_write_domain(obj);
2800 break;
2801 }
2802}
2803
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002804/**
2805 * Moves a single object to the GTT read, and possibly write domain.
2806 *
2807 * This function returns when the move is complete, including waiting on
2808 * flushes to occur.
2809 */
Jesse Barnes79e53942008-11-07 14:24:08 -08002810int
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002811i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2812{
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002813 struct drm_i915_gem_object *obj_priv = obj->driver_private;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002814 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08002815 int ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002816
Eric Anholt02354392008-11-26 13:58:13 -08002817 /* Not valid to be called on unbound objects. */
2818 if (obj_priv->gtt_space == NULL)
2819 return -EINVAL;
2820
Eric Anholte47c68e2008-11-14 13:35:19 -08002821 i915_gem_object_flush_gpu_write_domain(obj);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002822 /* Wait on any GPU rendering and flushing to occur. */
Eric Anholte47c68e2008-11-14 13:35:19 -08002823 ret = i915_gem_object_wait_rendering(obj);
2824 if (ret != 0)
2825 return ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002826
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002827 old_write_domain = obj->write_domain;
2828 old_read_domains = obj->read_domains;
2829
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002830 /* If we're writing through the GTT domain, then CPU and GPU caches
2831 * will need to be invalidated at next use.
2832 */
2833 if (write)
Eric Anholte47c68e2008-11-14 13:35:19 -08002834 obj->read_domains &= I915_GEM_DOMAIN_GTT;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002835
Eric Anholte47c68e2008-11-14 13:35:19 -08002836 i915_gem_object_flush_cpu_write_domain(obj);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002837
2838 /* It should now be out of any other write domains, and we can update
2839 * the domain values for our changes.
2840 */
2841 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2842 obj->read_domains |= I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08002843 if (write) {
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002844 obj->write_domain = I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08002845 obj_priv->dirty = 1;
2846 }
2847
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002848 trace_i915_gem_object_change_domain(obj,
2849 old_read_domains,
2850 old_write_domain);
2851
Eric Anholte47c68e2008-11-14 13:35:19 -08002852 return 0;
2853}
2854
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002855/*
2856 * Prepare buffer for display plane. Use uninterruptible for possible flush
2857 * wait, as in modesetting process we're not supposed to be interrupted.
2858 */
2859int
2860i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2861{
2862 struct drm_device *dev = obj->dev;
2863 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2864 uint32_t old_write_domain, old_read_domains;
2865 int ret;
2866
2867 /* Not valid to be called on unbound objects. */
2868 if (obj_priv->gtt_space == NULL)
2869 return -EINVAL;
2870
2871 i915_gem_object_flush_gpu_write_domain(obj);
2872
2873 /* Wait on any GPU rendering and flushing to occur. */
2874 if (obj_priv->active) {
2875#if WATCH_BUF
2876 DRM_INFO("%s: object %p wait for seqno %08x\n",
2877 __func__, obj, obj_priv->last_rendering_seqno);
2878#endif
2879 ret = i915_do_wait_request(dev, obj_priv->last_rendering_seqno, 0);
2880 if (ret != 0)
2881 return ret;
2882 }
2883
2884 old_write_domain = obj->write_domain;
2885 old_read_domains = obj->read_domains;
2886
2887 obj->read_domains &= I915_GEM_DOMAIN_GTT;
2888
2889 i915_gem_object_flush_cpu_write_domain(obj);
2890
2891 /* It should now be out of any other write domains, and we can update
2892 * the domain values for our changes.
2893 */
2894 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2895 obj->read_domains |= I915_GEM_DOMAIN_GTT;
2896 obj->write_domain = I915_GEM_DOMAIN_GTT;
2897 obj_priv->dirty = 1;
2898
2899 trace_i915_gem_object_change_domain(obj,
2900 old_read_domains,
2901 old_write_domain);
2902
2903 return 0;
2904}
2905
Eric Anholte47c68e2008-11-14 13:35:19 -08002906/**
2907 * Moves a single object to the CPU read, and possibly write domain.
2908 *
2909 * This function returns when the move is complete, including waiting on
2910 * flushes to occur.
2911 */
2912static int
2913i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2914{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002915 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08002916 int ret;
2917
2918 i915_gem_object_flush_gpu_write_domain(obj);
2919 /* Wait on any GPU rendering and flushing to occur. */
2920 ret = i915_gem_object_wait_rendering(obj);
2921 if (ret != 0)
2922 return ret;
2923
2924 i915_gem_object_flush_gtt_write_domain(obj);
2925
2926 /* If we have a partially-valid cache of the object in the CPU,
2927 * finish invalidating it and free the per-page flags.
2928 */
2929 i915_gem_object_set_to_full_cpu_read_domain(obj);
2930
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002931 old_write_domain = obj->write_domain;
2932 old_read_domains = obj->read_domains;
2933
Eric Anholte47c68e2008-11-14 13:35:19 -08002934 /* Flush the CPU cache if it's still invalid. */
2935 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2936 i915_gem_clflush_object(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08002937
2938 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2939 }
2940
2941 /* It should now be out of any other write domains, and we can update
2942 * the domain values for our changes.
2943 */
2944 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2945
2946 /* If we're writing through the CPU, then the GPU read domains will
2947 * need to be invalidated at next use.
2948 */
2949 if (write) {
2950 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2951 obj->write_domain = I915_GEM_DOMAIN_CPU;
2952 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002953
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002954 trace_i915_gem_object_change_domain(obj,
2955 old_read_domains,
2956 old_write_domain);
2957
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002958 return 0;
2959}
2960
Eric Anholt673a3942008-07-30 12:06:12 -07002961/*
2962 * Set the next domain for the specified object. This
2963 * may not actually perform the necessary flushing/invaliding though,
2964 * as that may want to be batched with other set_domain operations
2965 *
2966 * This is (we hope) the only really tricky part of gem. The goal
2967 * is fairly simple -- track which caches hold bits of the object
2968 * and make sure they remain coherent. A few concrete examples may
2969 * help to explain how it works. For shorthand, we use the notation
2970 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2971 * a pair of read and write domain masks.
2972 *
2973 * Case 1: the batch buffer
2974 *
2975 * 1. Allocated
2976 * 2. Written by CPU
2977 * 3. Mapped to GTT
2978 * 4. Read by GPU
2979 * 5. Unmapped from GTT
2980 * 6. Freed
2981 *
2982 * Let's take these a step at a time
2983 *
2984 * 1. Allocated
2985 * Pages allocated from the kernel may still have
2986 * cache contents, so we set them to (CPU, CPU) always.
2987 * 2. Written by CPU (using pwrite)
2988 * The pwrite function calls set_domain (CPU, CPU) and
2989 * this function does nothing (as nothing changes)
2990 * 3. Mapped by GTT
2991 * This function asserts that the object is not
2992 * currently in any GPU-based read or write domains
2993 * 4. Read by GPU
2994 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2995 * As write_domain is zero, this function adds in the
2996 * current read domains (CPU+COMMAND, 0).
2997 * flush_domains is set to CPU.
2998 * invalidate_domains is set to COMMAND
2999 * clflush is run to get data out of the CPU caches
3000 * then i915_dev_set_domain calls i915_gem_flush to
3001 * emit an MI_FLUSH and drm_agp_chipset_flush
3002 * 5. Unmapped from GTT
3003 * i915_gem_object_unbind calls set_domain (CPU, CPU)
3004 * flush_domains and invalidate_domains end up both zero
3005 * so no flushing/invalidating happens
3006 * 6. Freed
3007 * yay, done
3008 *
3009 * Case 2: The shared render buffer
3010 *
3011 * 1. Allocated
3012 * 2. Mapped to GTT
3013 * 3. Read/written by GPU
3014 * 4. set_domain to (CPU,CPU)
3015 * 5. Read/written by CPU
3016 * 6. Read/written by GPU
3017 *
3018 * 1. Allocated
3019 * Same as last example, (CPU, CPU)
3020 * 2. Mapped to GTT
3021 * Nothing changes (assertions find that it is not in the GPU)
3022 * 3. Read/written by GPU
3023 * execbuffer calls set_domain (RENDER, RENDER)
3024 * flush_domains gets CPU
3025 * invalidate_domains gets GPU
3026 * clflush (obj)
3027 * MI_FLUSH and drm_agp_chipset_flush
3028 * 4. set_domain (CPU, CPU)
3029 * flush_domains gets GPU
3030 * invalidate_domains gets CPU
3031 * wait_rendering (obj) to make sure all drawing is complete.
3032 * This will include an MI_FLUSH to get the data from GPU
3033 * to memory
3034 * clflush (obj) to invalidate the CPU cache
3035 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3036 * 5. Read/written by CPU
3037 * cache lines are loaded and dirtied
3038 * 6. Read written by GPU
3039 * Same as last GPU access
3040 *
3041 * Case 3: The constant buffer
3042 *
3043 * 1. Allocated
3044 * 2. Written by CPU
3045 * 3. Read by GPU
3046 * 4. Updated (written) by CPU again
3047 * 5. Read by GPU
3048 *
3049 * 1. Allocated
3050 * (CPU, CPU)
3051 * 2. Written by CPU
3052 * (CPU, CPU)
3053 * 3. Read by GPU
3054 * (CPU+RENDER, 0)
3055 * flush_domains = CPU
3056 * invalidate_domains = RENDER
3057 * clflush (obj)
3058 * MI_FLUSH
3059 * drm_agp_chipset_flush
3060 * 4. Updated (written) by CPU again
3061 * (CPU, CPU)
3062 * flush_domains = 0 (no previous write domain)
3063 * invalidate_domains = 0 (no new read domains)
3064 * 5. Read by GPU
3065 * (CPU+RENDER, 0)
3066 * flush_domains = CPU
3067 * invalidate_domains = RENDER
3068 * clflush (obj)
3069 * MI_FLUSH
3070 * drm_agp_chipset_flush
3071 */
Keith Packardc0d90822008-11-20 23:11:08 -08003072static void
Eric Anholt8b0e3782009-02-19 14:40:50 -08003073i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07003074{
3075 struct drm_device *dev = obj->dev;
3076 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3077 uint32_t invalidate_domains = 0;
3078 uint32_t flush_domains = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003079 uint32_t old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003080
Eric Anholt8b0e3782009-02-19 14:40:50 -08003081 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3082 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
Eric Anholt673a3942008-07-30 12:06:12 -07003083
Jesse Barnes652c3932009-08-17 13:31:43 -07003084 intel_mark_busy(dev, obj);
3085
Eric Anholt673a3942008-07-30 12:06:12 -07003086#if WATCH_BUF
3087 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
3088 __func__, obj,
Eric Anholt8b0e3782009-02-19 14:40:50 -08003089 obj->read_domains, obj->pending_read_domains,
3090 obj->write_domain, obj->pending_write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07003091#endif
3092 /*
3093 * If the object isn't moving to a new write domain,
3094 * let the object stay in multiple read domains
3095 */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003096 if (obj->pending_write_domain == 0)
3097 obj->pending_read_domains |= obj->read_domains;
Eric Anholt673a3942008-07-30 12:06:12 -07003098 else
3099 obj_priv->dirty = 1;
3100
3101 /*
3102 * Flush the current write domain if
3103 * the new read domains don't match. Invalidate
3104 * any read domains which differ from the old
3105 * write domain
3106 */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003107 if (obj->write_domain &&
3108 obj->write_domain != obj->pending_read_domains) {
Eric Anholt673a3942008-07-30 12:06:12 -07003109 flush_domains |= obj->write_domain;
Eric Anholt8b0e3782009-02-19 14:40:50 -08003110 invalidate_domains |=
3111 obj->pending_read_domains & ~obj->write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -07003112 }
3113 /*
3114 * Invalidate any read caches which may have
3115 * stale data. That is, any new read domains.
3116 */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003117 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
Eric Anholt673a3942008-07-30 12:06:12 -07003118 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3119#if WATCH_BUF
3120 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3121 __func__, flush_domains, invalidate_domains);
3122#endif
Eric Anholt673a3942008-07-30 12:06:12 -07003123 i915_gem_clflush_object(obj);
3124 }
3125
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003126 old_read_domains = obj->read_domains;
3127
Eric Anholtefbeed92009-02-19 14:54:51 -08003128 /* The actual obj->write_domain will be updated with
3129 * pending_write_domain after we emit the accumulated flush for all
3130 * of our domain changes in execbuffers (which clears objects'
3131 * write_domains). So if we have a current write domain that we
3132 * aren't changing, set pending_write_domain to that.
3133 */
3134 if (flush_domains == 0 && obj->pending_write_domain == 0)
3135 obj->pending_write_domain = obj->write_domain;
Eric Anholt8b0e3782009-02-19 14:40:50 -08003136 obj->read_domains = obj->pending_read_domains;
Eric Anholt673a3942008-07-30 12:06:12 -07003137
3138 dev->invalidate_domains |= invalidate_domains;
3139 dev->flush_domains |= flush_domains;
3140#if WATCH_BUF
3141 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3142 __func__,
3143 obj->read_domains, obj->write_domain,
3144 dev->invalidate_domains, dev->flush_domains);
3145#endif
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003146
3147 trace_i915_gem_object_change_domain(obj,
3148 old_read_domains,
3149 obj->write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07003150}
3151
3152/**
Eric Anholte47c68e2008-11-14 13:35:19 -08003153 * Moves the object from a partially CPU read to a full one.
Eric Anholt673a3942008-07-30 12:06:12 -07003154 *
Eric Anholte47c68e2008-11-14 13:35:19 -08003155 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3156 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3157 */
3158static void
3159i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3160{
Eric Anholte47c68e2008-11-14 13:35:19 -08003161 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3162
3163 if (!obj_priv->page_cpu_valid)
3164 return;
3165
3166 /* If we're partially in the CPU read domain, finish moving it in.
3167 */
3168 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3169 int i;
3170
3171 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3172 if (obj_priv->page_cpu_valid[i])
3173 continue;
Eric Anholt856fa192009-03-19 14:10:50 -07003174 drm_clflush_pages(obj_priv->pages + i, 1);
Eric Anholte47c68e2008-11-14 13:35:19 -08003175 }
Eric Anholte47c68e2008-11-14 13:35:19 -08003176 }
3177
3178 /* Free the page_cpu_valid mappings which are now stale, whether
3179 * or not we've got I915_GEM_DOMAIN_CPU.
3180 */
Eric Anholt9a298b22009-03-24 12:23:04 -07003181 kfree(obj_priv->page_cpu_valid);
Eric Anholte47c68e2008-11-14 13:35:19 -08003182 obj_priv->page_cpu_valid = NULL;
3183}
3184
3185/**
3186 * Set the CPU read domain on a range of the object.
3187 *
3188 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3189 * not entirely valid. The page_cpu_valid member of the object flags which
3190 * pages have been flushed, and will be respected by
3191 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3192 * of the whole object.
3193 *
3194 * This function returns when the move is complete, including waiting on
3195 * flushes to occur.
Eric Anholt673a3942008-07-30 12:06:12 -07003196 */
3197static int
Eric Anholte47c68e2008-11-14 13:35:19 -08003198i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3199 uint64_t offset, uint64_t size)
Eric Anholt673a3942008-07-30 12:06:12 -07003200{
3201 struct drm_i915_gem_object *obj_priv = obj->driver_private;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003202 uint32_t old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003203 int i, ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003204
Eric Anholte47c68e2008-11-14 13:35:19 -08003205 if (offset == 0 && size == obj->size)
3206 return i915_gem_object_set_to_cpu_domain(obj, 0);
3207
3208 i915_gem_object_flush_gpu_write_domain(obj);
3209 /* Wait on any GPU rendering and flushing to occur. */
3210 ret = i915_gem_object_wait_rendering(obj);
3211 if (ret != 0)
3212 return ret;
3213 i915_gem_object_flush_gtt_write_domain(obj);
3214
3215 /* If we're already fully in the CPU read domain, we're done. */
3216 if (obj_priv->page_cpu_valid == NULL &&
3217 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07003218 return 0;
3219
Eric Anholte47c68e2008-11-14 13:35:19 -08003220 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3221 * newly adding I915_GEM_DOMAIN_CPU
3222 */
Eric Anholt673a3942008-07-30 12:06:12 -07003223 if (obj_priv->page_cpu_valid == NULL) {
Eric Anholt9a298b22009-03-24 12:23:04 -07003224 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3225 GFP_KERNEL);
Eric Anholte47c68e2008-11-14 13:35:19 -08003226 if (obj_priv->page_cpu_valid == NULL)
3227 return -ENOMEM;
3228 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3229 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
Eric Anholt673a3942008-07-30 12:06:12 -07003230
3231 /* Flush the cache on any pages that are still invalid from the CPU's
3232 * perspective.
3233 */
Eric Anholte47c68e2008-11-14 13:35:19 -08003234 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3235 i++) {
Eric Anholt673a3942008-07-30 12:06:12 -07003236 if (obj_priv->page_cpu_valid[i])
3237 continue;
3238
Eric Anholt856fa192009-03-19 14:10:50 -07003239 drm_clflush_pages(obj_priv->pages + i, 1);
Eric Anholt673a3942008-07-30 12:06:12 -07003240
3241 obj_priv->page_cpu_valid[i] = 1;
3242 }
3243
Eric Anholte47c68e2008-11-14 13:35:19 -08003244 /* It should now be out of any other write domains, and we can update
3245 * the domain values for our changes.
3246 */
3247 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3248
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003249 old_read_domains = obj->read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003250 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3251
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003252 trace_i915_gem_object_change_domain(obj,
3253 old_read_domains,
3254 obj->write_domain);
3255
Eric Anholt673a3942008-07-30 12:06:12 -07003256 return 0;
3257}
3258
3259/**
Eric Anholt673a3942008-07-30 12:06:12 -07003260 * Pin an object to the GTT and evaluate the relocations landing in it.
3261 */
3262static int
3263i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3264 struct drm_file *file_priv,
Jesse Barnes76446ca2009-12-17 22:05:42 -05003265 struct drm_i915_gem_exec_object2 *entry,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003266 struct drm_i915_gem_relocation_entry *relocs)
Eric Anholt673a3942008-07-30 12:06:12 -07003267{
3268 struct drm_device *dev = obj->dev;
Keith Packard0839ccb2008-10-30 19:38:48 -07003269 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07003270 struct drm_i915_gem_object *obj_priv = obj->driver_private;
3271 int i, ret;
Keith Packard0839ccb2008-10-30 19:38:48 -07003272 void __iomem *reloc_page;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003273 bool need_fence;
3274
3275 need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3276 obj_priv->tiling_mode != I915_TILING_NONE;
3277
3278 /* Check fence reg constraints and rebind if necessary */
Owain Ainsworthf590d272010-02-18 15:33:00 +00003279 if (need_fence && !i915_gem_object_fence_offset_ok(obj,
3280 obj_priv->tiling_mode))
Jesse Barnes76446ca2009-12-17 22:05:42 -05003281 i915_gem_object_unbind(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003282
3283 /* Choose the GTT offset for our buffer and put it there. */
3284 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3285 if (ret)
3286 return ret;
3287
Jesse Barnes76446ca2009-12-17 22:05:42 -05003288 /*
3289 * Pre-965 chips need a fence register set up in order to
3290 * properly handle blits to/from tiled surfaces.
3291 */
3292 if (need_fence) {
3293 ret = i915_gem_object_get_fence_reg(obj);
3294 if (ret != 0) {
3295 if (ret != -EBUSY && ret != -ERESTARTSYS)
3296 DRM_ERROR("Failure to install fence: %d\n",
3297 ret);
3298 i915_gem_object_unpin(obj);
3299 return ret;
3300 }
3301 }
3302
Eric Anholt673a3942008-07-30 12:06:12 -07003303 entry->offset = obj_priv->gtt_offset;
3304
Eric Anholt673a3942008-07-30 12:06:12 -07003305 /* Apply the relocations, using the GTT aperture to avoid cache
3306 * flushing requirements.
3307 */
3308 for (i = 0; i < entry->relocation_count; i++) {
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003309 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
Eric Anholt673a3942008-07-30 12:06:12 -07003310 struct drm_gem_object *target_obj;
3311 struct drm_i915_gem_object *target_obj_priv;
Eric Anholt3043c602008-10-02 12:24:47 -07003312 uint32_t reloc_val, reloc_offset;
3313 uint32_t __iomem *reloc_entry;
Eric Anholt673a3942008-07-30 12:06:12 -07003314
Eric Anholt673a3942008-07-30 12:06:12 -07003315 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003316 reloc->target_handle);
Eric Anholt673a3942008-07-30 12:06:12 -07003317 if (target_obj == NULL) {
3318 i915_gem_object_unpin(obj);
3319 return -EBADF;
3320 }
3321 target_obj_priv = target_obj->driver_private;
3322
Chris Wilson8542a0b2009-09-09 21:15:15 +01003323#if WATCH_RELOC
3324 DRM_INFO("%s: obj %p offset %08x target %d "
3325 "read %08x write %08x gtt %08x "
3326 "presumed %08x delta %08x\n",
3327 __func__,
3328 obj,
3329 (int) reloc->offset,
3330 (int) reloc->target_handle,
3331 (int) reloc->read_domains,
3332 (int) reloc->write_domain,
3333 (int) target_obj_priv->gtt_offset,
3334 (int) reloc->presumed_offset,
3335 reloc->delta);
3336#endif
3337
Eric Anholt673a3942008-07-30 12:06:12 -07003338 /* The target buffer should have appeared before us in the
3339 * exec_object list, so it should have a GTT space bound by now.
3340 */
3341 if (target_obj_priv->gtt_space == NULL) {
3342 DRM_ERROR("No GTT space found for object %d\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003343 reloc->target_handle);
Eric Anholt673a3942008-07-30 12:06:12 -07003344 drm_gem_object_unreference(target_obj);
3345 i915_gem_object_unpin(obj);
3346 return -EINVAL;
3347 }
3348
Chris Wilson8542a0b2009-09-09 21:15:15 +01003349 /* Validate that the target is in a valid r/w GPU domain */
3350 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3351 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3352 DRM_ERROR("reloc with read/write CPU domains: "
3353 "obj %p target %d offset %d "
3354 "read %08x write %08x",
3355 obj, reloc->target_handle,
3356 (int) reloc->offset,
3357 reloc->read_domains,
3358 reloc->write_domain);
3359 drm_gem_object_unreference(target_obj);
3360 i915_gem_object_unpin(obj);
3361 return -EINVAL;
3362 }
3363 if (reloc->write_domain && target_obj->pending_write_domain &&
3364 reloc->write_domain != target_obj->pending_write_domain) {
3365 DRM_ERROR("Write domain conflict: "
3366 "obj %p target %d offset %d "
3367 "new %08x old %08x\n",
3368 obj, reloc->target_handle,
3369 (int) reloc->offset,
3370 reloc->write_domain,
3371 target_obj->pending_write_domain);
3372 drm_gem_object_unreference(target_obj);
3373 i915_gem_object_unpin(obj);
3374 return -EINVAL;
3375 }
3376
3377 target_obj->pending_read_domains |= reloc->read_domains;
3378 target_obj->pending_write_domain |= reloc->write_domain;
3379
3380 /* If the relocation already has the right value in it, no
3381 * more work needs to be done.
3382 */
3383 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3384 drm_gem_object_unreference(target_obj);
3385 continue;
3386 }
3387
3388 /* Check that the relocation address is valid... */
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003389 if (reloc->offset > obj->size - 4) {
Eric Anholt673a3942008-07-30 12:06:12 -07003390 DRM_ERROR("Relocation beyond object bounds: "
3391 "obj %p target %d offset %d size %d.\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003392 obj, reloc->target_handle,
3393 (int) reloc->offset, (int) obj->size);
Eric Anholt673a3942008-07-30 12:06:12 -07003394 drm_gem_object_unreference(target_obj);
3395 i915_gem_object_unpin(obj);
3396 return -EINVAL;
3397 }
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003398 if (reloc->offset & 3) {
Eric Anholt673a3942008-07-30 12:06:12 -07003399 DRM_ERROR("Relocation not 4-byte aligned: "
3400 "obj %p target %d offset %d.\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003401 obj, reloc->target_handle,
3402 (int) reloc->offset);
Eric Anholt673a3942008-07-30 12:06:12 -07003403 drm_gem_object_unreference(target_obj);
3404 i915_gem_object_unpin(obj);
3405 return -EINVAL;
3406 }
3407
Chris Wilson8542a0b2009-09-09 21:15:15 +01003408 /* and points to somewhere within the target object. */
Chris Wilsoncd0b9fb2009-09-15 23:23:18 +01003409 if (reloc->delta >= target_obj->size) {
3410 DRM_ERROR("Relocation beyond target object bounds: "
3411 "obj %p target %d delta %d size %d.\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003412 obj, reloc->target_handle,
Chris Wilsoncd0b9fb2009-09-15 23:23:18 +01003413 (int) reloc->delta, (int) target_obj->size);
Chris Wilson491152b2009-02-11 14:26:32 +00003414 drm_gem_object_unreference(target_obj);
3415 i915_gem_object_unpin(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08003416 return -EINVAL;
3417 }
3418
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003419 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3420 if (ret != 0) {
3421 drm_gem_object_unreference(target_obj);
3422 i915_gem_object_unpin(obj);
3423 return -EINVAL;
Eric Anholt673a3942008-07-30 12:06:12 -07003424 }
3425
3426 /* Map the page containing the relocation we're going to
3427 * perform.
3428 */
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003429 reloc_offset = obj_priv->gtt_offset + reloc->offset;
Keith Packard0839ccb2008-10-30 19:38:48 -07003430 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3431 (reloc_offset &
3432 ~(PAGE_SIZE - 1)));
Eric Anholt3043c602008-10-02 12:24:47 -07003433 reloc_entry = (uint32_t __iomem *)(reloc_page +
Keith Packard0839ccb2008-10-30 19:38:48 -07003434 (reloc_offset & (PAGE_SIZE - 1)));
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003435 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
Eric Anholt673a3942008-07-30 12:06:12 -07003436
3437#if WATCH_BUF
3438 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003439 obj, (unsigned int) reloc->offset,
Eric Anholt673a3942008-07-30 12:06:12 -07003440 readl(reloc_entry), reloc_val);
3441#endif
3442 writel(reloc_val, reloc_entry);
Keith Packard0839ccb2008-10-30 19:38:48 -07003443 io_mapping_unmap_atomic(reloc_page);
Eric Anholt673a3942008-07-30 12:06:12 -07003444
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003445 /* The updated presumed offset for this entry will be
3446 * copied back out to the user.
Eric Anholt673a3942008-07-30 12:06:12 -07003447 */
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003448 reloc->presumed_offset = target_obj_priv->gtt_offset;
Eric Anholt673a3942008-07-30 12:06:12 -07003449
3450 drm_gem_object_unreference(target_obj);
3451 }
3452
Eric Anholt673a3942008-07-30 12:06:12 -07003453#if WATCH_BUF
3454 if (0)
3455 i915_gem_dump_object(obj, 128, __func__, ~0);
3456#endif
3457 return 0;
3458}
3459
3460/** Dispatch a batchbuffer to the ring
3461 */
3462static int
3463i915_dispatch_gem_execbuffer(struct drm_device *dev,
Jesse Barnes76446ca2009-12-17 22:05:42 -05003464 struct drm_i915_gem_execbuffer2 *exec,
Eric Anholt201361a2009-03-11 12:30:04 -07003465 struct drm_clip_rect *cliprects,
Eric Anholt673a3942008-07-30 12:06:12 -07003466 uint64_t exec_offset)
3467{
3468 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07003469 int nbox = exec->num_cliprects;
3470 int i = 0, count;
Chris Wilson83d60792009-06-06 09:45:57 +01003471 uint32_t exec_start, exec_len;
Eric Anholt673a3942008-07-30 12:06:12 -07003472 RING_LOCALS;
3473
3474 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3475 exec_len = (uint32_t) exec->batch_len;
3476
Chris Wilson8f0dc5b2009-09-24 00:43:17 +01003477 trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003478
Eric Anholt673a3942008-07-30 12:06:12 -07003479 count = nbox ? nbox : 1;
3480
3481 for (i = 0; i < count; i++) {
3482 if (i < nbox) {
Eric Anholt201361a2009-03-11 12:30:04 -07003483 int ret = i915_emit_box(dev, cliprects, i,
Eric Anholt673a3942008-07-30 12:06:12 -07003484 exec->DR1, exec->DR4);
3485 if (ret)
3486 return ret;
3487 }
3488
3489 if (IS_I830(dev) || IS_845G(dev)) {
3490 BEGIN_LP_RING(4);
3491 OUT_RING(MI_BATCH_BUFFER);
3492 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3493 OUT_RING(exec_start + exec_len - 4);
3494 OUT_RING(0);
3495 ADVANCE_LP_RING();
3496 } else {
3497 BEGIN_LP_RING(2);
3498 if (IS_I965G(dev)) {
3499 OUT_RING(MI_BATCH_BUFFER_START |
3500 (2 << 6) |
3501 MI_BATCH_NON_SECURE_I965);
3502 OUT_RING(exec_start);
3503 } else {
3504 OUT_RING(MI_BATCH_BUFFER_START |
3505 (2 << 6));
3506 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
3507 }
3508 ADVANCE_LP_RING();
3509 }
3510 }
3511
3512 /* XXX breadcrumb */
3513 return 0;
3514}
3515
3516/* Throttle our rendering by waiting until the ring has completed our requests
3517 * emitted over 20 msec ago.
3518 *
Eric Anholtb9624422009-06-03 07:27:35 +00003519 * Note that if we were to use the current jiffies each time around the loop,
3520 * we wouldn't escape the function with any frames outstanding if the time to
3521 * render a frame was over 20ms.
3522 *
Eric Anholt673a3942008-07-30 12:06:12 -07003523 * This should get us reasonable parallelism between CPU and GPU but also
3524 * relatively low latency when blocking on a particular request to finish.
3525 */
3526static int
3527i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3528{
3529 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3530 int ret = 0;
Eric Anholtb9624422009-06-03 07:27:35 +00003531 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
Eric Anholt673a3942008-07-30 12:06:12 -07003532
3533 mutex_lock(&dev->struct_mutex);
Eric Anholtb9624422009-06-03 07:27:35 +00003534 while (!list_empty(&i915_file_priv->mm.request_list)) {
3535 struct drm_i915_gem_request *request;
3536
3537 request = list_first_entry(&i915_file_priv->mm.request_list,
3538 struct drm_i915_gem_request,
3539 client_list);
3540
3541 if (time_after_eq(request->emitted_jiffies, recent_enough))
3542 break;
3543
3544 ret = i915_wait_request(dev, request->seqno);
3545 if (ret != 0)
3546 break;
3547 }
Eric Anholt673a3942008-07-30 12:06:12 -07003548 mutex_unlock(&dev->struct_mutex);
Eric Anholtb9624422009-06-03 07:27:35 +00003549
Eric Anholt673a3942008-07-30 12:06:12 -07003550 return ret;
3551}
3552
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003553static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003554i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003555 uint32_t buffer_count,
3556 struct drm_i915_gem_relocation_entry **relocs)
3557{
3558 uint32_t reloc_count = 0, reloc_index = 0, i;
3559 int ret;
3560
3561 *relocs = NULL;
3562 for (i = 0; i < buffer_count; i++) {
3563 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3564 return -EINVAL;
3565 reloc_count += exec_list[i].relocation_count;
3566 }
3567
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003568 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
Jesse Barnes76446ca2009-12-17 22:05:42 -05003569 if (*relocs == NULL) {
3570 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003571 return -ENOMEM;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003572 }
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003573
3574 for (i = 0; i < buffer_count; i++) {
3575 struct drm_i915_gem_relocation_entry __user *user_relocs;
3576
3577 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3578
3579 ret = copy_from_user(&(*relocs)[reloc_index],
3580 user_relocs,
3581 exec_list[i].relocation_count *
3582 sizeof(**relocs));
3583 if (ret != 0) {
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003584 drm_free_large(*relocs);
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003585 *relocs = NULL;
Florian Mickler2bc43b52009-04-06 22:55:41 +02003586 return -EFAULT;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003587 }
3588
3589 reloc_index += exec_list[i].relocation_count;
3590 }
3591
Florian Mickler2bc43b52009-04-06 22:55:41 +02003592 return 0;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003593}
3594
3595static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003596i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003597 uint32_t buffer_count,
3598 struct drm_i915_gem_relocation_entry *relocs)
3599{
3600 uint32_t reloc_count = 0, i;
Florian Mickler2bc43b52009-04-06 22:55:41 +02003601 int ret = 0;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003602
Chris Wilson93533c22010-01-31 10:40:48 +00003603 if (relocs == NULL)
3604 return 0;
3605
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003606 for (i = 0; i < buffer_count; i++) {
3607 struct drm_i915_gem_relocation_entry __user *user_relocs;
Florian Mickler2bc43b52009-04-06 22:55:41 +02003608 int unwritten;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003609
3610 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3611
Florian Mickler2bc43b52009-04-06 22:55:41 +02003612 unwritten = copy_to_user(user_relocs,
3613 &relocs[reloc_count],
3614 exec_list[i].relocation_count *
3615 sizeof(*relocs));
3616
3617 if (unwritten) {
3618 ret = -EFAULT;
3619 goto err;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003620 }
3621
3622 reloc_count += exec_list[i].relocation_count;
3623 }
3624
Florian Mickler2bc43b52009-04-06 22:55:41 +02003625err:
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003626 drm_free_large(relocs);
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003627
3628 return ret;
3629}
3630
Chris Wilson83d60792009-06-06 09:45:57 +01003631static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003632i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
Chris Wilson83d60792009-06-06 09:45:57 +01003633 uint64_t exec_offset)
3634{
3635 uint32_t exec_start, exec_len;
3636
3637 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3638 exec_len = (uint32_t) exec->batch_len;
3639
3640 if ((exec_start | exec_len) & 0x7)
3641 return -EINVAL;
3642
3643 if (!exec_start)
3644 return -EINVAL;
3645
3646 return 0;
3647}
3648
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003649static int
3650i915_gem_wait_for_pending_flip(struct drm_device *dev,
3651 struct drm_gem_object **object_list,
3652 int count)
3653{
3654 drm_i915_private_t *dev_priv = dev->dev_private;
3655 struct drm_i915_gem_object *obj_priv;
3656 DEFINE_WAIT(wait);
3657 int i, ret = 0;
3658
3659 for (;;) {
3660 prepare_to_wait(&dev_priv->pending_flip_queue,
3661 &wait, TASK_INTERRUPTIBLE);
3662 for (i = 0; i < count; i++) {
3663 obj_priv = object_list[i]->driver_private;
3664 if (atomic_read(&obj_priv->pending_flip) > 0)
3665 break;
3666 }
3667 if (i == count)
3668 break;
3669
3670 if (!signal_pending(current)) {
3671 mutex_unlock(&dev->struct_mutex);
3672 schedule();
3673 mutex_lock(&dev->struct_mutex);
3674 continue;
3675 }
3676 ret = -ERESTARTSYS;
3677 break;
3678 }
3679 finish_wait(&dev_priv->pending_flip_queue, &wait);
3680
3681 return ret;
3682}
3683
Eric Anholt673a3942008-07-30 12:06:12 -07003684int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003685i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3686 struct drm_file *file_priv,
3687 struct drm_i915_gem_execbuffer2 *args,
3688 struct drm_i915_gem_exec_object2 *exec_list)
Eric Anholt673a3942008-07-30 12:06:12 -07003689{
3690 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07003691 struct drm_gem_object **object_list = NULL;
3692 struct drm_gem_object *batch_obj;
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003693 struct drm_i915_gem_object *obj_priv;
Eric Anholt201361a2009-03-11 12:30:04 -07003694 struct drm_clip_rect *cliprects = NULL;
Chris Wilson93533c22010-01-31 10:40:48 +00003695 struct drm_i915_gem_relocation_entry *relocs = NULL;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003696 int ret = 0, ret2, i, pinned = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07003697 uint64_t exec_offset;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003698 uint32_t seqno, flush_domains, reloc_index;
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003699 int pin_tries, flips;
Eric Anholt673a3942008-07-30 12:06:12 -07003700
3701#if WATCH_EXEC
3702 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3703 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3704#endif
3705
Eric Anholt4f481ed2008-09-10 14:22:49 -07003706 if (args->buffer_count < 1) {
3707 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3708 return -EINVAL;
3709 }
Eric Anholtc8e0f932009-11-22 03:49:37 +01003710 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
Jesse Barnes76446ca2009-12-17 22:05:42 -05003711 if (object_list == NULL) {
3712 DRM_ERROR("Failed to allocate object list for %d buffers\n",
Eric Anholt673a3942008-07-30 12:06:12 -07003713 args->buffer_count);
3714 ret = -ENOMEM;
3715 goto pre_mutex_err;
3716 }
Eric Anholt673a3942008-07-30 12:06:12 -07003717
Eric Anholt201361a2009-03-11 12:30:04 -07003718 if (args->num_cliprects != 0) {
Eric Anholt9a298b22009-03-24 12:23:04 -07003719 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3720 GFP_KERNEL);
Owain Ainswortha40e8d32010-02-09 14:25:55 +00003721 if (cliprects == NULL) {
3722 ret = -ENOMEM;
Eric Anholt201361a2009-03-11 12:30:04 -07003723 goto pre_mutex_err;
Owain Ainswortha40e8d32010-02-09 14:25:55 +00003724 }
Eric Anholt201361a2009-03-11 12:30:04 -07003725
3726 ret = copy_from_user(cliprects,
3727 (struct drm_clip_rect __user *)
3728 (uintptr_t) args->cliprects_ptr,
3729 sizeof(*cliprects) * args->num_cliprects);
3730 if (ret != 0) {
3731 DRM_ERROR("copy %d cliprects failed: %d\n",
3732 args->num_cliprects, ret);
3733 goto pre_mutex_err;
3734 }
3735 }
3736
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003737 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3738 &relocs);
3739 if (ret != 0)
3740 goto pre_mutex_err;
3741
Eric Anholt673a3942008-07-30 12:06:12 -07003742 mutex_lock(&dev->struct_mutex);
3743
3744 i915_verify_inactive(dev, __FILE__, __LINE__);
3745
Ben Gamariba1234d2009-09-14 17:48:47 -04003746 if (atomic_read(&dev_priv->mm.wedged)) {
Eric Anholt673a3942008-07-30 12:06:12 -07003747 mutex_unlock(&dev->struct_mutex);
Chris Wilsona198bc82009-02-06 16:55:20 +00003748 ret = -EIO;
3749 goto pre_mutex_err;
Eric Anholt673a3942008-07-30 12:06:12 -07003750 }
3751
3752 if (dev_priv->mm.suspended) {
Eric Anholt673a3942008-07-30 12:06:12 -07003753 mutex_unlock(&dev->struct_mutex);
Chris Wilsona198bc82009-02-06 16:55:20 +00003754 ret = -EBUSY;
3755 goto pre_mutex_err;
Eric Anholt673a3942008-07-30 12:06:12 -07003756 }
3757
Keith Packardac94a962008-11-20 23:30:27 -08003758 /* Look up object handles */
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003759 flips = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07003760 for (i = 0; i < args->buffer_count; i++) {
3761 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3762 exec_list[i].handle);
3763 if (object_list[i] == NULL) {
3764 DRM_ERROR("Invalid object handle %d at index %d\n",
3765 exec_list[i].handle, i);
Chris Wilson0ce907f2010-01-23 20:26:35 +00003766 /* prevent error path from reading uninitialized data */
3767 args->buffer_count = i + 1;
Eric Anholt673a3942008-07-30 12:06:12 -07003768 ret = -EBADF;
3769 goto err;
3770 }
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003771
3772 obj_priv = object_list[i]->driver_private;
3773 if (obj_priv->in_execbuffer) {
3774 DRM_ERROR("Object %p appears more than once in object list\n",
3775 object_list[i]);
Chris Wilson0ce907f2010-01-23 20:26:35 +00003776 /* prevent error path from reading uninitialized data */
3777 args->buffer_count = i + 1;
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003778 ret = -EBADF;
3779 goto err;
3780 }
3781 obj_priv->in_execbuffer = true;
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003782 flips += atomic_read(&obj_priv->pending_flip);
3783 }
3784
3785 if (flips > 0) {
3786 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3787 args->buffer_count);
3788 if (ret)
3789 goto err;
Keith Packardac94a962008-11-20 23:30:27 -08003790 }
Eric Anholt673a3942008-07-30 12:06:12 -07003791
Keith Packardac94a962008-11-20 23:30:27 -08003792 /* Pin and relocate */
3793 for (pin_tries = 0; ; pin_tries++) {
3794 ret = 0;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003795 reloc_index = 0;
3796
Keith Packardac94a962008-11-20 23:30:27 -08003797 for (i = 0; i < args->buffer_count; i++) {
3798 object_list[i]->pending_read_domains = 0;
3799 object_list[i]->pending_write_domain = 0;
3800 ret = i915_gem_object_pin_and_relocate(object_list[i],
3801 file_priv,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003802 &exec_list[i],
3803 &relocs[reloc_index]);
Keith Packardac94a962008-11-20 23:30:27 -08003804 if (ret)
3805 break;
3806 pinned = i + 1;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003807 reloc_index += exec_list[i].relocation_count;
Keith Packardac94a962008-11-20 23:30:27 -08003808 }
3809 /* success */
3810 if (ret == 0)
3811 break;
3812
3813 /* error other than GTT full, or we've already tried again */
Chris Wilson2939e1f2009-06-06 09:46:03 +01003814 if (ret != -ENOSPC || pin_tries >= 1) {
Chris Wilson07f73f62009-09-14 16:50:30 +01003815 if (ret != -ERESTARTSYS) {
3816 unsigned long long total_size = 0;
3817 for (i = 0; i < args->buffer_count; i++)
3818 total_size += object_list[i]->size;
3819 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes: %d\n",
3820 pinned+1, args->buffer_count,
3821 total_size, ret);
3822 DRM_ERROR("%d objects [%d pinned], "
3823 "%d object bytes [%d pinned], "
3824 "%d/%d gtt bytes\n",
3825 atomic_read(&dev->object_count),
3826 atomic_read(&dev->pin_count),
3827 atomic_read(&dev->object_memory),
3828 atomic_read(&dev->pin_memory),
3829 atomic_read(&dev->gtt_memory),
3830 dev->gtt_total);
3831 }
Eric Anholt673a3942008-07-30 12:06:12 -07003832 goto err;
3833 }
Keith Packardac94a962008-11-20 23:30:27 -08003834
3835 /* unpin all of our buffers */
3836 for (i = 0; i < pinned; i++)
3837 i915_gem_object_unpin(object_list[i]);
Eric Anholtb1177632008-12-10 10:09:41 -08003838 pinned = 0;
Keith Packardac94a962008-11-20 23:30:27 -08003839
3840 /* evict everyone we can from the aperture */
3841 ret = i915_gem_evict_everything(dev);
Chris Wilson07f73f62009-09-14 16:50:30 +01003842 if (ret && ret != -ENOSPC)
Keith Packardac94a962008-11-20 23:30:27 -08003843 goto err;
Eric Anholt673a3942008-07-30 12:06:12 -07003844 }
3845
3846 /* Set the pending read domains for the batch buffer to COMMAND */
3847 batch_obj = object_list[args->buffer_count-1];
Chris Wilson5f26a2c2009-06-06 09:45:58 +01003848 if (batch_obj->pending_write_domain) {
3849 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3850 ret = -EINVAL;
3851 goto err;
3852 }
3853 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
Eric Anholt673a3942008-07-30 12:06:12 -07003854
Chris Wilson83d60792009-06-06 09:45:57 +01003855 /* Sanity check the batch buffer, prior to moving objects */
3856 exec_offset = exec_list[args->buffer_count - 1].offset;
3857 ret = i915_gem_check_execbuffer (args, exec_offset);
3858 if (ret != 0) {
3859 DRM_ERROR("execbuf with invalid offset/length\n");
3860 goto err;
3861 }
3862
Eric Anholt673a3942008-07-30 12:06:12 -07003863 i915_verify_inactive(dev, __FILE__, __LINE__);
3864
Keith Packard646f0f62008-11-20 23:23:03 -08003865 /* Zero the global flush/invalidate flags. These
3866 * will be modified as new domains are computed
3867 * for each object
3868 */
3869 dev->invalidate_domains = 0;
3870 dev->flush_domains = 0;
3871
Eric Anholt673a3942008-07-30 12:06:12 -07003872 for (i = 0; i < args->buffer_count; i++) {
3873 struct drm_gem_object *obj = object_list[i];
Eric Anholt673a3942008-07-30 12:06:12 -07003874
Keith Packard646f0f62008-11-20 23:23:03 -08003875 /* Compute new gpu domains and update invalidate/flush */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003876 i915_gem_object_set_to_gpu_domain(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003877 }
3878
3879 i915_verify_inactive(dev, __FILE__, __LINE__);
3880
Keith Packard646f0f62008-11-20 23:23:03 -08003881 if (dev->invalidate_domains | dev->flush_domains) {
3882#if WATCH_EXEC
3883 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3884 __func__,
3885 dev->invalidate_domains,
3886 dev->flush_domains);
3887#endif
3888 i915_gem_flush(dev,
3889 dev->invalidate_domains,
3890 dev->flush_domains);
Daniel Vetter99fcb762010-02-07 16:20:18 +01003891 if (dev->flush_domains & I915_GEM_GPU_DOMAINS)
Eric Anholtb9624422009-06-03 07:27:35 +00003892 (void)i915_add_request(dev, file_priv,
3893 dev->flush_domains);
Keith Packard646f0f62008-11-20 23:23:03 -08003894 }
Eric Anholt673a3942008-07-30 12:06:12 -07003895
Eric Anholtefbeed92009-02-19 14:54:51 -08003896 for (i = 0; i < args->buffer_count; i++) {
3897 struct drm_gem_object *obj = object_list[i];
Daniel Vetter99fcb762010-02-07 16:20:18 +01003898 struct drm_i915_gem_object *obj_priv = obj->driver_private;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003899 uint32_t old_write_domain = obj->write_domain;
Eric Anholtefbeed92009-02-19 14:54:51 -08003900
3901 obj->write_domain = obj->pending_write_domain;
Daniel Vetter99fcb762010-02-07 16:20:18 +01003902 if (obj->write_domain)
3903 list_move_tail(&obj_priv->gpu_write_list,
3904 &dev_priv->mm.gpu_write_list);
3905 else
3906 list_del_init(&obj_priv->gpu_write_list);
3907
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003908 trace_i915_gem_object_change_domain(obj,
3909 obj->read_domains,
3910 old_write_domain);
Eric Anholtefbeed92009-02-19 14:54:51 -08003911 }
3912
Eric Anholt673a3942008-07-30 12:06:12 -07003913 i915_verify_inactive(dev, __FILE__, __LINE__);
3914
3915#if WATCH_COHERENCY
3916 for (i = 0; i < args->buffer_count; i++) {
3917 i915_gem_object_check_coherency(object_list[i],
3918 exec_list[i].handle);
3919 }
3920#endif
3921
Eric Anholt673a3942008-07-30 12:06:12 -07003922#if WATCH_EXEC
Ben Gamari6911a9b2009-04-02 11:24:54 -07003923 i915_gem_dump_object(batch_obj,
Eric Anholt673a3942008-07-30 12:06:12 -07003924 args->batch_len,
3925 __func__,
3926 ~0);
3927#endif
3928
Eric Anholt673a3942008-07-30 12:06:12 -07003929 /* Exec the batchbuffer */
Eric Anholt201361a2009-03-11 12:30:04 -07003930 ret = i915_dispatch_gem_execbuffer(dev, args, cliprects, exec_offset);
Eric Anholt673a3942008-07-30 12:06:12 -07003931 if (ret) {
3932 DRM_ERROR("dispatch failed %d\n", ret);
3933 goto err;
3934 }
3935
3936 /*
3937 * Ensure that the commands in the batch buffer are
3938 * finished before the interrupt fires
3939 */
3940 flush_domains = i915_retire_commands(dev);
3941
3942 i915_verify_inactive(dev, __FILE__, __LINE__);
3943
3944 /*
3945 * Get a seqno representing the execution of the current buffer,
3946 * which we can wait on. We would like to mitigate these interrupts,
3947 * likely by only creating seqnos occasionally (so that we have
3948 * *some* interrupts representing completion of buffers that we can
3949 * wait on when trying to clear up gtt space).
3950 */
Eric Anholtb9624422009-06-03 07:27:35 +00003951 seqno = i915_add_request(dev, file_priv, flush_domains);
Eric Anholt673a3942008-07-30 12:06:12 -07003952 BUG_ON(seqno == 0);
Eric Anholt673a3942008-07-30 12:06:12 -07003953 for (i = 0; i < args->buffer_count; i++) {
3954 struct drm_gem_object *obj = object_list[i];
Eric Anholt673a3942008-07-30 12:06:12 -07003955
Eric Anholtce44b0e2008-11-06 16:00:31 -08003956 i915_gem_object_move_to_active(obj, seqno);
Eric Anholt673a3942008-07-30 12:06:12 -07003957#if WATCH_LRU
3958 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3959#endif
3960 }
3961#if WATCH_LRU
3962 i915_dump_lru(dev, __func__);
3963#endif
3964
3965 i915_verify_inactive(dev, __FILE__, __LINE__);
3966
Eric Anholt673a3942008-07-30 12:06:12 -07003967err:
Julia Lawallaad87df2008-12-21 16:28:47 +01003968 for (i = 0; i < pinned; i++)
3969 i915_gem_object_unpin(object_list[i]);
Eric Anholt673a3942008-07-30 12:06:12 -07003970
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003971 for (i = 0; i < args->buffer_count; i++) {
3972 if (object_list[i]) {
3973 obj_priv = object_list[i]->driver_private;
3974 obj_priv->in_execbuffer = false;
3975 }
Julia Lawallaad87df2008-12-21 16:28:47 +01003976 drm_gem_object_unreference(object_list[i]);
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003977 }
Julia Lawallaad87df2008-12-21 16:28:47 +01003978
Eric Anholt673a3942008-07-30 12:06:12 -07003979 mutex_unlock(&dev->struct_mutex);
3980
Chris Wilson93533c22010-01-31 10:40:48 +00003981pre_mutex_err:
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003982 /* Copy the updated relocations out regardless of current error
3983 * state. Failure to update the relocs would mean that the next
3984 * time userland calls execbuf, it would do so with presumed offset
3985 * state that didn't match the actual object state.
3986 */
3987 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3988 relocs);
3989 if (ret2 != 0) {
3990 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3991
3992 if (ret == 0)
3993 ret = ret2;
3994 }
3995
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003996 drm_free_large(object_list);
Eric Anholt9a298b22009-03-24 12:23:04 -07003997 kfree(cliprects);
Eric Anholt673a3942008-07-30 12:06:12 -07003998
3999 return ret;
4000}
4001
Jesse Barnes76446ca2009-12-17 22:05:42 -05004002/*
4003 * Legacy execbuffer just creates an exec2 list from the original exec object
4004 * list array and passes it to the real function.
4005 */
4006int
4007i915_gem_execbuffer(struct drm_device *dev, void *data,
4008 struct drm_file *file_priv)
4009{
4010 struct drm_i915_gem_execbuffer *args = data;
4011 struct drm_i915_gem_execbuffer2 exec2;
4012 struct drm_i915_gem_exec_object *exec_list = NULL;
4013 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4014 int ret, i;
4015
4016#if WATCH_EXEC
4017 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4018 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4019#endif
4020
4021 if (args->buffer_count < 1) {
4022 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4023 return -EINVAL;
4024 }
4025
4026 /* Copy in the exec list from userland */
4027 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4028 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4029 if (exec_list == NULL || exec2_list == NULL) {
4030 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4031 args->buffer_count);
4032 drm_free_large(exec_list);
4033 drm_free_large(exec2_list);
4034 return -ENOMEM;
4035 }
4036 ret = copy_from_user(exec_list,
4037 (struct drm_i915_relocation_entry __user *)
4038 (uintptr_t) args->buffers_ptr,
4039 sizeof(*exec_list) * args->buffer_count);
4040 if (ret != 0) {
4041 DRM_ERROR("copy %d exec entries failed %d\n",
4042 args->buffer_count, ret);
4043 drm_free_large(exec_list);
4044 drm_free_large(exec2_list);
4045 return -EFAULT;
4046 }
4047
4048 for (i = 0; i < args->buffer_count; i++) {
4049 exec2_list[i].handle = exec_list[i].handle;
4050 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4051 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4052 exec2_list[i].alignment = exec_list[i].alignment;
4053 exec2_list[i].offset = exec_list[i].offset;
4054 if (!IS_I965G(dev))
4055 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4056 else
4057 exec2_list[i].flags = 0;
4058 }
4059
4060 exec2.buffers_ptr = args->buffers_ptr;
4061 exec2.buffer_count = args->buffer_count;
4062 exec2.batch_start_offset = args->batch_start_offset;
4063 exec2.batch_len = args->batch_len;
4064 exec2.DR1 = args->DR1;
4065 exec2.DR4 = args->DR4;
4066 exec2.num_cliprects = args->num_cliprects;
4067 exec2.cliprects_ptr = args->cliprects_ptr;
4068 exec2.flags = 0;
4069
4070 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4071 if (!ret) {
4072 /* Copy the new buffer offsets back to the user's exec list. */
4073 for (i = 0; i < args->buffer_count; i++)
4074 exec_list[i].offset = exec2_list[i].offset;
4075 /* ... and back out to userspace */
4076 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4077 (uintptr_t) args->buffers_ptr,
4078 exec_list,
4079 sizeof(*exec_list) * args->buffer_count);
4080 if (ret) {
4081 ret = -EFAULT;
4082 DRM_ERROR("failed to copy %d exec entries "
4083 "back to user (%d)\n",
4084 args->buffer_count, ret);
4085 }
Jesse Barnes76446ca2009-12-17 22:05:42 -05004086 }
4087
4088 drm_free_large(exec_list);
4089 drm_free_large(exec2_list);
4090 return ret;
4091}
4092
4093int
4094i915_gem_execbuffer2(struct drm_device *dev, void *data,
4095 struct drm_file *file_priv)
4096{
4097 struct drm_i915_gem_execbuffer2 *args = data;
4098 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4099 int ret;
4100
4101#if WATCH_EXEC
4102 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4103 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4104#endif
4105
4106 if (args->buffer_count < 1) {
4107 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4108 return -EINVAL;
4109 }
4110
4111 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4112 if (exec2_list == NULL) {
4113 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4114 args->buffer_count);
4115 return -ENOMEM;
4116 }
4117 ret = copy_from_user(exec2_list,
4118 (struct drm_i915_relocation_entry __user *)
4119 (uintptr_t) args->buffers_ptr,
4120 sizeof(*exec2_list) * args->buffer_count);
4121 if (ret != 0) {
4122 DRM_ERROR("copy %d exec entries failed %d\n",
4123 args->buffer_count, ret);
4124 drm_free_large(exec2_list);
4125 return -EFAULT;
4126 }
4127
4128 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4129 if (!ret) {
4130 /* Copy the new buffer offsets back to the user's exec list. */
4131 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4132 (uintptr_t) args->buffers_ptr,
4133 exec2_list,
4134 sizeof(*exec2_list) * args->buffer_count);
4135 if (ret) {
4136 ret = -EFAULT;
4137 DRM_ERROR("failed to copy %d exec entries "
4138 "back to user (%d)\n",
4139 args->buffer_count, ret);
4140 }
4141 }
4142
4143 drm_free_large(exec2_list);
4144 return ret;
4145}
4146
Eric Anholt673a3942008-07-30 12:06:12 -07004147int
4148i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4149{
4150 struct drm_device *dev = obj->dev;
4151 struct drm_i915_gem_object *obj_priv = obj->driver_private;
4152 int ret;
4153
4154 i915_verify_inactive(dev, __FILE__, __LINE__);
4155 if (obj_priv->gtt_space == NULL) {
4156 ret = i915_gem_object_bind_to_gtt(obj, alignment);
Chris Wilson97311292009-09-21 00:22:34 +01004157 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07004158 return ret;
Chris Wilson22c344e2009-02-11 14:26:45 +00004159 }
Jesse Barnes76446ca2009-12-17 22:05:42 -05004160
Eric Anholt673a3942008-07-30 12:06:12 -07004161 obj_priv->pin_count++;
4162
4163 /* If the object is not active and not pending a flush,
4164 * remove it from the inactive list
4165 */
4166 if (obj_priv->pin_count == 1) {
4167 atomic_inc(&dev->pin_count);
4168 atomic_add(obj->size, &dev->pin_memory);
4169 if (!obj_priv->active &&
Chris Wilson21d509e2009-06-06 09:46:02 +01004170 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
Eric Anholt673a3942008-07-30 12:06:12 -07004171 !list_empty(&obj_priv->list))
4172 list_del_init(&obj_priv->list);
4173 }
4174 i915_verify_inactive(dev, __FILE__, __LINE__);
4175
4176 return 0;
4177}
4178
4179void
4180i915_gem_object_unpin(struct drm_gem_object *obj)
4181{
4182 struct drm_device *dev = obj->dev;
4183 drm_i915_private_t *dev_priv = dev->dev_private;
4184 struct drm_i915_gem_object *obj_priv = obj->driver_private;
4185
4186 i915_verify_inactive(dev, __FILE__, __LINE__);
4187 obj_priv->pin_count--;
4188 BUG_ON(obj_priv->pin_count < 0);
4189 BUG_ON(obj_priv->gtt_space == NULL);
4190
4191 /* If the object is no longer pinned, and is
4192 * neither active nor being flushed, then stick it on
4193 * the inactive list
4194 */
4195 if (obj_priv->pin_count == 0) {
4196 if (!obj_priv->active &&
Chris Wilson21d509e2009-06-06 09:46:02 +01004197 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
Eric Anholt673a3942008-07-30 12:06:12 -07004198 list_move_tail(&obj_priv->list,
4199 &dev_priv->mm.inactive_list);
4200 atomic_dec(&dev->pin_count);
4201 atomic_sub(obj->size, &dev->pin_memory);
4202 }
4203 i915_verify_inactive(dev, __FILE__, __LINE__);
4204}
4205
4206int
4207i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4208 struct drm_file *file_priv)
4209{
4210 struct drm_i915_gem_pin *args = data;
4211 struct drm_gem_object *obj;
4212 struct drm_i915_gem_object *obj_priv;
4213 int ret;
4214
4215 mutex_lock(&dev->struct_mutex);
4216
4217 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4218 if (obj == NULL) {
4219 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4220 args->handle);
4221 mutex_unlock(&dev->struct_mutex);
4222 return -EBADF;
4223 }
4224 obj_priv = obj->driver_private;
4225
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004226 if (obj_priv->madv != I915_MADV_WILLNEED) {
4227 DRM_ERROR("Attempting to pin a purgeable buffer\n");
Chris Wilson3ef94da2009-09-14 16:50:29 +01004228 drm_gem_object_unreference(obj);
4229 mutex_unlock(&dev->struct_mutex);
4230 return -EINVAL;
4231 }
4232
Jesse Barnes79e53942008-11-07 14:24:08 -08004233 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4234 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4235 args->handle);
Chris Wilson96dec612009-02-08 19:08:04 +00004236 drm_gem_object_unreference(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004237 mutex_unlock(&dev->struct_mutex);
Jesse Barnes79e53942008-11-07 14:24:08 -08004238 return -EINVAL;
4239 }
4240
4241 obj_priv->user_pin_count++;
4242 obj_priv->pin_filp = file_priv;
4243 if (obj_priv->user_pin_count == 1) {
4244 ret = i915_gem_object_pin(obj, args->alignment);
4245 if (ret != 0) {
4246 drm_gem_object_unreference(obj);
4247 mutex_unlock(&dev->struct_mutex);
4248 return ret;
4249 }
Eric Anholt673a3942008-07-30 12:06:12 -07004250 }
4251
4252 /* XXX - flush the CPU caches for pinned objects
4253 * as the X server doesn't manage domains yet
4254 */
Eric Anholte47c68e2008-11-14 13:35:19 -08004255 i915_gem_object_flush_cpu_write_domain(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004256 args->offset = obj_priv->gtt_offset;
4257 drm_gem_object_unreference(obj);
4258 mutex_unlock(&dev->struct_mutex);
4259
4260 return 0;
4261}
4262
4263int
4264i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4265 struct drm_file *file_priv)
4266{
4267 struct drm_i915_gem_pin *args = data;
4268 struct drm_gem_object *obj;
Jesse Barnes79e53942008-11-07 14:24:08 -08004269 struct drm_i915_gem_object *obj_priv;
Eric Anholt673a3942008-07-30 12:06:12 -07004270
4271 mutex_lock(&dev->struct_mutex);
4272
4273 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4274 if (obj == NULL) {
4275 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4276 args->handle);
4277 mutex_unlock(&dev->struct_mutex);
4278 return -EBADF;
4279 }
4280
Jesse Barnes79e53942008-11-07 14:24:08 -08004281 obj_priv = obj->driver_private;
4282 if (obj_priv->pin_filp != file_priv) {
4283 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4284 args->handle);
4285 drm_gem_object_unreference(obj);
4286 mutex_unlock(&dev->struct_mutex);
4287 return -EINVAL;
4288 }
4289 obj_priv->user_pin_count--;
4290 if (obj_priv->user_pin_count == 0) {
4291 obj_priv->pin_filp = NULL;
4292 i915_gem_object_unpin(obj);
4293 }
Eric Anholt673a3942008-07-30 12:06:12 -07004294
4295 drm_gem_object_unreference(obj);
4296 mutex_unlock(&dev->struct_mutex);
4297 return 0;
4298}
4299
4300int
4301i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4302 struct drm_file *file_priv)
4303{
4304 struct drm_i915_gem_busy *args = data;
4305 struct drm_gem_object *obj;
4306 struct drm_i915_gem_object *obj_priv;
4307
Eric Anholt673a3942008-07-30 12:06:12 -07004308 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4309 if (obj == NULL) {
4310 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4311 args->handle);
Eric Anholt673a3942008-07-30 12:06:12 -07004312 return -EBADF;
4313 }
4314
Chris Wilsonb1ce7862009-06-06 09:46:00 +01004315 mutex_lock(&dev->struct_mutex);
Eric Anholtf21289b2009-02-18 09:44:56 -08004316 /* Update the active list for the hardware's current position.
4317 * Otherwise this only updates on a delayed timer or when irqs are
4318 * actually unmasked, and our working set ends up being larger than
4319 * required.
4320 */
4321 i915_gem_retire_requests(dev);
4322
Eric Anholt673a3942008-07-30 12:06:12 -07004323 obj_priv = obj->driver_private;
Eric Anholtc4de0a52008-12-14 19:05:04 -08004324 /* Don't count being on the flushing list against the object being
4325 * done. Otherwise, a buffer left on the flushing list but not getting
4326 * flushed (because nobody's flushing that domain) won't ever return
4327 * unbusy and get reused by libdrm's bo cache. The other expected
4328 * consumer of this interface, OpenGL's occlusion queries, also specs
4329 * that the objects get unbusy "eventually" without any interference.
4330 */
4331 args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
Eric Anholt673a3942008-07-30 12:06:12 -07004332
4333 drm_gem_object_unreference(obj);
4334 mutex_unlock(&dev->struct_mutex);
4335 return 0;
4336}
4337
4338int
4339i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4340 struct drm_file *file_priv)
4341{
4342 return i915_gem_ring_throttle(dev, file_priv);
4343}
4344
Chris Wilson3ef94da2009-09-14 16:50:29 +01004345int
4346i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4347 struct drm_file *file_priv)
4348{
4349 struct drm_i915_gem_madvise *args = data;
4350 struct drm_gem_object *obj;
4351 struct drm_i915_gem_object *obj_priv;
4352
4353 switch (args->madv) {
4354 case I915_MADV_DONTNEED:
4355 case I915_MADV_WILLNEED:
4356 break;
4357 default:
4358 return -EINVAL;
4359 }
4360
4361 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4362 if (obj == NULL) {
4363 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4364 args->handle);
4365 return -EBADF;
4366 }
4367
4368 mutex_lock(&dev->struct_mutex);
4369 obj_priv = obj->driver_private;
4370
4371 if (obj_priv->pin_count) {
4372 drm_gem_object_unreference(obj);
4373 mutex_unlock(&dev->struct_mutex);
4374
4375 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4376 return -EINVAL;
4377 }
4378
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004379 if (obj_priv->madv != __I915_MADV_PURGED)
4380 obj_priv->madv = args->madv;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004381
Chris Wilson2d7ef392009-09-20 23:13:10 +01004382 /* if the object is no longer bound, discard its backing storage */
4383 if (i915_gem_object_is_purgeable(obj_priv) &&
4384 obj_priv->gtt_space == NULL)
4385 i915_gem_object_truncate(obj);
4386
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004387 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4388
Chris Wilson3ef94da2009-09-14 16:50:29 +01004389 drm_gem_object_unreference(obj);
4390 mutex_unlock(&dev->struct_mutex);
4391
4392 return 0;
4393}
4394
Eric Anholt673a3942008-07-30 12:06:12 -07004395int i915_gem_init_object(struct drm_gem_object *obj)
4396{
4397 struct drm_i915_gem_object *obj_priv;
4398
Eric Anholt9a298b22009-03-24 12:23:04 -07004399 obj_priv = kzalloc(sizeof(*obj_priv), GFP_KERNEL);
Eric Anholt673a3942008-07-30 12:06:12 -07004400 if (obj_priv == NULL)
4401 return -ENOMEM;
4402
4403 /*
4404 * We've just allocated pages from the kernel,
4405 * so they've just been written by the CPU with
4406 * zeros. They'll need to be clflushed before we
4407 * use them with the GPU.
4408 */
4409 obj->write_domain = I915_GEM_DOMAIN_CPU;
4410 obj->read_domains = I915_GEM_DOMAIN_CPU;
4411
Keith Packardba1eb1d2008-10-14 19:55:10 -07004412 obj_priv->agp_type = AGP_USER_MEMORY;
4413
Eric Anholt673a3942008-07-30 12:06:12 -07004414 obj->driver_private = obj_priv;
4415 obj_priv->obj = obj;
Jesse Barnesde151cf2008-11-12 10:03:55 -08004416 obj_priv->fence_reg = I915_FENCE_REG_NONE;
Eric Anholt673a3942008-07-30 12:06:12 -07004417 INIT_LIST_HEAD(&obj_priv->list);
Daniel Vetter99fcb762010-02-07 16:20:18 +01004418 INIT_LIST_HEAD(&obj_priv->gpu_write_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07004419 INIT_LIST_HEAD(&obj_priv->fence_list);
Chris Wilson3ef94da2009-09-14 16:50:29 +01004420 obj_priv->madv = I915_MADV_WILLNEED;
Jesse Barnesde151cf2008-11-12 10:03:55 -08004421
Chris Wilson1c5d22f2009-08-25 11:15:50 +01004422 trace_i915_gem_object_create(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004423
4424 return 0;
4425}
4426
4427void i915_gem_free_object(struct drm_gem_object *obj)
4428{
Jesse Barnesde151cf2008-11-12 10:03:55 -08004429 struct drm_device *dev = obj->dev;
Eric Anholt673a3942008-07-30 12:06:12 -07004430 struct drm_i915_gem_object *obj_priv = obj->driver_private;
4431
Chris Wilson1c5d22f2009-08-25 11:15:50 +01004432 trace_i915_gem_object_destroy(obj);
4433
Eric Anholt673a3942008-07-30 12:06:12 -07004434 while (obj_priv->pin_count > 0)
4435 i915_gem_object_unpin(obj);
4436
Dave Airlie71acb5e2008-12-30 20:31:46 +10004437 if (obj_priv->phys_obj)
4438 i915_gem_detach_phys_object(dev, obj);
4439
Eric Anholt673a3942008-07-30 12:06:12 -07004440 i915_gem_object_unbind(obj);
4441
Chris Wilson7e616152009-09-10 08:53:04 +01004442 if (obj_priv->mmap_offset)
4443 i915_gem_free_mmap_offset(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08004444
Eric Anholt9a298b22009-03-24 12:23:04 -07004445 kfree(obj_priv->page_cpu_valid);
Eric Anholt280b7132009-03-12 16:56:27 -07004446 kfree(obj_priv->bit_17);
Eric Anholt9a298b22009-03-24 12:23:04 -07004447 kfree(obj->driver_private);
Eric Anholt673a3942008-07-30 12:06:12 -07004448}
4449
Chris Wilsonab5ee572009-09-20 19:25:47 +01004450/** Unbinds all inactive objects. */
Eric Anholt673a3942008-07-30 12:06:12 -07004451static int
Chris Wilsonab5ee572009-09-20 19:25:47 +01004452i915_gem_evict_from_inactive_list(struct drm_device *dev)
Eric Anholt673a3942008-07-30 12:06:12 -07004453{
Chris Wilsonab5ee572009-09-20 19:25:47 +01004454 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07004455
Chris Wilsonab5ee572009-09-20 19:25:47 +01004456 while (!list_empty(&dev_priv->mm.inactive_list)) {
4457 struct drm_gem_object *obj;
4458 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004459
Chris Wilsonab5ee572009-09-20 19:25:47 +01004460 obj = list_first_entry(&dev_priv->mm.inactive_list,
4461 struct drm_i915_gem_object,
4462 list)->obj;
Eric Anholt673a3942008-07-30 12:06:12 -07004463
4464 ret = i915_gem_object_unbind(obj);
4465 if (ret != 0) {
Chris Wilsonab5ee572009-09-20 19:25:47 +01004466 DRM_ERROR("Error unbinding object: %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07004467 return ret;
4468 }
4469 }
4470
Eric Anholt673a3942008-07-30 12:06:12 -07004471 return 0;
4472}
4473
Chris Wilson29105cc2010-01-07 10:39:13 +00004474static int
4475i915_gpu_idle(struct drm_device *dev)
4476{
4477 drm_i915_private_t *dev_priv = dev->dev_private;
4478 bool lists_empty;
4479 uint32_t seqno;
4480
4481 spin_lock(&dev_priv->mm.active_list_lock);
4482 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4483 list_empty(&dev_priv->mm.active_list);
4484 spin_unlock(&dev_priv->mm.active_list_lock);
4485
4486 if (lists_empty)
4487 return 0;
4488
4489 /* Flush everything onto the inactive list. */
4490 i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
4491 seqno = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS);
4492 if (seqno == 0)
4493 return -ENOMEM;
4494
4495 return i915_wait_request(dev, seqno);
4496}
4497
Jesse Barnes5669fca2009-02-17 15:13:31 -08004498int
Eric Anholt673a3942008-07-30 12:06:12 -07004499i915_gem_idle(struct drm_device *dev)
4500{
4501 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson29105cc2010-01-07 10:39:13 +00004502 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004503
Keith Packard6dbe2772008-10-14 21:41:13 -07004504 mutex_lock(&dev->struct_mutex);
4505
4506 if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
4507 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07004508 return 0;
Keith Packard6dbe2772008-10-14 21:41:13 -07004509 }
Eric Anholt673a3942008-07-30 12:06:12 -07004510
Chris Wilson29105cc2010-01-07 10:39:13 +00004511 ret = i915_gpu_idle(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07004512 if (ret) {
4513 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07004514 return ret;
Keith Packard6dbe2772008-10-14 21:41:13 -07004515 }
Eric Anholt673a3942008-07-30 12:06:12 -07004516
Chris Wilson29105cc2010-01-07 10:39:13 +00004517 /* Under UMS, be paranoid and evict. */
4518 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4519 ret = i915_gem_evict_from_inactive_list(dev);
4520 if (ret) {
4521 mutex_unlock(&dev->struct_mutex);
4522 return ret;
4523 }
4524 }
4525
4526 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4527 * We need to replace this with a semaphore, or something.
4528 * And not confound mm.suspended!
4529 */
4530 dev_priv->mm.suspended = 1;
4531 del_timer(&dev_priv->hangcheck_timer);
4532
4533 i915_kernel_lost_context(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07004534 i915_gem_cleanup_ringbuffer(dev);
Chris Wilson29105cc2010-01-07 10:39:13 +00004535
Keith Packard6dbe2772008-10-14 21:41:13 -07004536 mutex_unlock(&dev->struct_mutex);
4537
Chris Wilson29105cc2010-01-07 10:39:13 +00004538 /* Cancel the retire work handler, which should be idle now. */
4539 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4540
Eric Anholt673a3942008-07-30 12:06:12 -07004541 return 0;
4542}
4543
4544static int
4545i915_gem_init_hws(struct drm_device *dev)
4546{
4547 drm_i915_private_t *dev_priv = dev->dev_private;
4548 struct drm_gem_object *obj;
4549 struct drm_i915_gem_object *obj_priv;
4550 int ret;
4551
4552 /* If we need a physical address for the status page, it's already
4553 * initialized at driver load time.
4554 */
4555 if (!I915_NEED_GFX_HWS(dev))
4556 return 0;
4557
4558 obj = drm_gem_object_alloc(dev, 4096);
4559 if (obj == NULL) {
4560 DRM_ERROR("Failed to allocate status page\n");
4561 return -ENOMEM;
4562 }
4563 obj_priv = obj->driver_private;
Keith Packardba1eb1d2008-10-14 19:55:10 -07004564 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
Eric Anholt673a3942008-07-30 12:06:12 -07004565
4566 ret = i915_gem_object_pin(obj, 4096);
4567 if (ret != 0) {
4568 drm_gem_object_unreference(obj);
4569 return ret;
4570 }
4571
4572 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
Eric Anholt673a3942008-07-30 12:06:12 -07004573
Eric Anholt856fa192009-03-19 14:10:50 -07004574 dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
Keith Packardba1eb1d2008-10-14 19:55:10 -07004575 if (dev_priv->hw_status_page == NULL) {
Eric Anholt673a3942008-07-30 12:06:12 -07004576 DRM_ERROR("Failed to map status page.\n");
4577 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
Chris Wilson3eb2ee72009-02-11 14:26:34 +00004578 i915_gem_object_unpin(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004579 drm_gem_object_unreference(obj);
4580 return -EINVAL;
4581 }
4582 dev_priv->hws_obj = obj;
Eric Anholt673a3942008-07-30 12:06:12 -07004583 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
4584 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
Keith Packardba1eb1d2008-10-14 19:55:10 -07004585 I915_READ(HWS_PGA); /* posting read */
Zhao Yakui44d98a62009-10-09 11:39:40 +08004586 DRM_DEBUG_DRIVER("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
Eric Anholt673a3942008-07-30 12:06:12 -07004587
4588 return 0;
4589}
4590
Chris Wilson85a7bb92009-02-11 14:52:44 +00004591static void
4592i915_gem_cleanup_hws(struct drm_device *dev)
4593{
4594 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsonbab2d1f2009-02-20 17:52:20 +00004595 struct drm_gem_object *obj;
4596 struct drm_i915_gem_object *obj_priv;
Chris Wilson85a7bb92009-02-11 14:52:44 +00004597
4598 if (dev_priv->hws_obj == NULL)
4599 return;
4600
Chris Wilsonbab2d1f2009-02-20 17:52:20 +00004601 obj = dev_priv->hws_obj;
4602 obj_priv = obj->driver_private;
4603
Eric Anholt856fa192009-03-19 14:10:50 -07004604 kunmap(obj_priv->pages[0]);
Chris Wilson85a7bb92009-02-11 14:52:44 +00004605 i915_gem_object_unpin(obj);
4606 drm_gem_object_unreference(obj);
4607 dev_priv->hws_obj = NULL;
Chris Wilsonbab2d1f2009-02-20 17:52:20 +00004608
Chris Wilson85a7bb92009-02-11 14:52:44 +00004609 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
4610 dev_priv->hw_status_page = NULL;
4611
4612 /* Write high address into HWS_PGA when disabling. */
4613 I915_WRITE(HWS_PGA, 0x1ffff000);
4614}
4615
Jesse Barnes79e53942008-11-07 14:24:08 -08004616int
Eric Anholt673a3942008-07-30 12:06:12 -07004617i915_gem_init_ringbuffer(struct drm_device *dev)
4618{
4619 drm_i915_private_t *dev_priv = dev->dev_private;
4620 struct drm_gem_object *obj;
4621 struct drm_i915_gem_object *obj_priv;
Jesse Barnes79e53942008-11-07 14:24:08 -08004622 drm_i915_ring_buffer_t *ring = &dev_priv->ring;
Eric Anholt673a3942008-07-30 12:06:12 -07004623 int ret;
Keith Packard50aa253d2008-10-14 17:20:35 -07004624 u32 head;
Eric Anholt673a3942008-07-30 12:06:12 -07004625
4626 ret = i915_gem_init_hws(dev);
4627 if (ret != 0)
4628 return ret;
4629
4630 obj = drm_gem_object_alloc(dev, 128 * 1024);
4631 if (obj == NULL) {
4632 DRM_ERROR("Failed to allocate ringbuffer\n");
Chris Wilson85a7bb92009-02-11 14:52:44 +00004633 i915_gem_cleanup_hws(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07004634 return -ENOMEM;
4635 }
4636 obj_priv = obj->driver_private;
4637
4638 ret = i915_gem_object_pin(obj, 4096);
4639 if (ret != 0) {
4640 drm_gem_object_unreference(obj);
Chris Wilson85a7bb92009-02-11 14:52:44 +00004641 i915_gem_cleanup_hws(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07004642 return ret;
4643 }
4644
4645 /* Set up the kernel mapping for the ring. */
Jesse Barnes79e53942008-11-07 14:24:08 -08004646 ring->Size = obj->size;
Eric Anholt673a3942008-07-30 12:06:12 -07004647
Jesse Barnes79e53942008-11-07 14:24:08 -08004648 ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
4649 ring->map.size = obj->size;
4650 ring->map.type = 0;
4651 ring->map.flags = 0;
4652 ring->map.mtrr = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07004653
Jesse Barnes79e53942008-11-07 14:24:08 -08004654 drm_core_ioremap_wc(&ring->map, dev);
4655 if (ring->map.handle == NULL) {
Eric Anholt673a3942008-07-30 12:06:12 -07004656 DRM_ERROR("Failed to map ringbuffer.\n");
4657 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
Chris Wilson47ed1852009-02-11 14:26:33 +00004658 i915_gem_object_unpin(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004659 drm_gem_object_unreference(obj);
Chris Wilson85a7bb92009-02-11 14:52:44 +00004660 i915_gem_cleanup_hws(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07004661 return -EINVAL;
4662 }
Jesse Barnes79e53942008-11-07 14:24:08 -08004663 ring->ring_obj = obj;
4664 ring->virtual_start = ring->map.handle;
Eric Anholt673a3942008-07-30 12:06:12 -07004665
4666 /* Stop the ring if it's running. */
4667 I915_WRITE(PRB0_CTL, 0);
Eric Anholt673a3942008-07-30 12:06:12 -07004668 I915_WRITE(PRB0_TAIL, 0);
Keith Packard50aa253d2008-10-14 17:20:35 -07004669 I915_WRITE(PRB0_HEAD, 0);
Eric Anholt673a3942008-07-30 12:06:12 -07004670
4671 /* Initialize the ring. */
4672 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
Keith Packard50aa253d2008-10-14 17:20:35 -07004673 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4674
4675 /* G45 ring initialization fails to reset head to zero */
4676 if (head != 0) {
4677 DRM_ERROR("Ring head not reset to zero "
4678 "ctl %08x head %08x tail %08x start %08x\n",
4679 I915_READ(PRB0_CTL),
4680 I915_READ(PRB0_HEAD),
4681 I915_READ(PRB0_TAIL),
4682 I915_READ(PRB0_START));
4683 I915_WRITE(PRB0_HEAD, 0);
4684
4685 DRM_ERROR("Ring head forced to zero "
4686 "ctl %08x head %08x tail %08x start %08x\n",
4687 I915_READ(PRB0_CTL),
4688 I915_READ(PRB0_HEAD),
4689 I915_READ(PRB0_TAIL),
4690 I915_READ(PRB0_START));
4691 }
4692
Eric Anholt673a3942008-07-30 12:06:12 -07004693 I915_WRITE(PRB0_CTL,
4694 ((obj->size - 4096) & RING_NR_PAGES) |
4695 RING_NO_REPORT |
4696 RING_VALID);
4697
Keith Packard50aa253d2008-10-14 17:20:35 -07004698 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4699
4700 /* If the head is still not zero, the ring is dead */
4701 if (head != 0) {
4702 DRM_ERROR("Ring initialization failed "
4703 "ctl %08x head %08x tail %08x start %08x\n",
4704 I915_READ(PRB0_CTL),
4705 I915_READ(PRB0_HEAD),
4706 I915_READ(PRB0_TAIL),
4707 I915_READ(PRB0_START));
4708 return -EIO;
4709 }
4710
Eric Anholt673a3942008-07-30 12:06:12 -07004711 /* Update our cache of the ring state */
Jesse Barnes79e53942008-11-07 14:24:08 -08004712 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4713 i915_kernel_lost_context(dev);
4714 else {
4715 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
4716 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
4717 ring->space = ring->head - (ring->tail + 8);
4718 if (ring->space < 0)
4719 ring->space += ring->Size;
4720 }
Eric Anholt673a3942008-07-30 12:06:12 -07004721
4722 return 0;
4723}
4724
Jesse Barnes79e53942008-11-07 14:24:08 -08004725void
Eric Anholt673a3942008-07-30 12:06:12 -07004726i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4727{
4728 drm_i915_private_t *dev_priv = dev->dev_private;
4729
4730 if (dev_priv->ring.ring_obj == NULL)
4731 return;
4732
4733 drm_core_ioremapfree(&dev_priv->ring.map, dev);
4734
4735 i915_gem_object_unpin(dev_priv->ring.ring_obj);
4736 drm_gem_object_unreference(dev_priv->ring.ring_obj);
4737 dev_priv->ring.ring_obj = NULL;
4738 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
4739
Chris Wilson85a7bb92009-02-11 14:52:44 +00004740 i915_gem_cleanup_hws(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07004741}
4742
4743int
4744i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4745 struct drm_file *file_priv)
4746{
4747 drm_i915_private_t *dev_priv = dev->dev_private;
4748 int ret;
4749
Jesse Barnes79e53942008-11-07 14:24:08 -08004750 if (drm_core_check_feature(dev, DRIVER_MODESET))
4751 return 0;
4752
Ben Gamariba1234d2009-09-14 17:48:47 -04004753 if (atomic_read(&dev_priv->mm.wedged)) {
Eric Anholt673a3942008-07-30 12:06:12 -07004754 DRM_ERROR("Reenabling wedged hardware, good luck\n");
Ben Gamariba1234d2009-09-14 17:48:47 -04004755 atomic_set(&dev_priv->mm.wedged, 0);
Eric Anholt673a3942008-07-30 12:06:12 -07004756 }
4757
Eric Anholt673a3942008-07-30 12:06:12 -07004758 mutex_lock(&dev->struct_mutex);
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004759 dev_priv->mm.suspended = 0;
4760
4761 ret = i915_gem_init_ringbuffer(dev);
Wu Fengguangd816f6a2009-04-18 10:43:32 +08004762 if (ret != 0) {
4763 mutex_unlock(&dev->struct_mutex);
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004764 return ret;
Wu Fengguangd816f6a2009-04-18 10:43:32 +08004765 }
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004766
Carl Worth5e118f42009-03-20 11:54:25 -07004767 spin_lock(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07004768 BUG_ON(!list_empty(&dev_priv->mm.active_list));
Carl Worth5e118f42009-03-20 11:54:25 -07004769 spin_unlock(&dev_priv->mm.active_list_lock);
4770
Eric Anholt673a3942008-07-30 12:06:12 -07004771 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4772 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
4773 BUG_ON(!list_empty(&dev_priv->mm.request_list));
Eric Anholt673a3942008-07-30 12:06:12 -07004774 mutex_unlock(&dev->struct_mutex);
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004775
4776 drm_irq_install(dev);
4777
Eric Anholt673a3942008-07-30 12:06:12 -07004778 return 0;
4779}
4780
4781int
4782i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4783 struct drm_file *file_priv)
4784{
Jesse Barnes79e53942008-11-07 14:24:08 -08004785 if (drm_core_check_feature(dev, DRIVER_MODESET))
4786 return 0;
4787
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004788 drm_irq_uninstall(dev);
Linus Torvaldse6890f62009-09-08 17:09:24 -07004789 return i915_gem_idle(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07004790}
4791
4792void
4793i915_gem_lastclose(struct drm_device *dev)
4794{
4795 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004796
Eric Anholte806b492009-01-22 09:56:58 -08004797 if (drm_core_check_feature(dev, DRIVER_MODESET))
4798 return;
4799
Keith Packard6dbe2772008-10-14 21:41:13 -07004800 ret = i915_gem_idle(dev);
4801 if (ret)
4802 DRM_ERROR("failed to idle hardware: %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07004803}
4804
4805void
4806i915_gem_load(struct drm_device *dev)
4807{
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004808 int i;
Eric Anholt673a3942008-07-30 12:06:12 -07004809 drm_i915_private_t *dev_priv = dev->dev_private;
4810
Carl Worth5e118f42009-03-20 11:54:25 -07004811 spin_lock_init(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07004812 INIT_LIST_HEAD(&dev_priv->mm.active_list);
4813 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
Daniel Vetter99fcb762010-02-07 16:20:18 +01004814 INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004815 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
4816 INIT_LIST_HEAD(&dev_priv->mm.request_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07004817 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004818 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4819 i915_gem_retire_work_handler);
Eric Anholt673a3942008-07-30 12:06:12 -07004820 dev_priv->mm.next_gem_seqno = 1;
4821
Chris Wilson31169712009-09-14 16:50:28 +01004822 spin_lock(&shrink_list_lock);
4823 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4824 spin_unlock(&shrink_list_lock);
4825
Jesse Barnesde151cf2008-11-12 10:03:55 -08004826 /* Old X drivers will take 0-2 for front, back, depth buffers */
Eric Anholtb397c832010-01-26 09:43:10 -08004827 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4828 dev_priv->fence_reg_start = 3;
Jesse Barnesde151cf2008-11-12 10:03:55 -08004829
Jesse Barnes0f973f22009-01-26 17:10:45 -08004830 if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
Jesse Barnesde151cf2008-11-12 10:03:55 -08004831 dev_priv->num_fence_regs = 16;
4832 else
4833 dev_priv->num_fence_regs = 8;
4834
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004835 /* Initialize fence registers to zero */
4836 if (IS_I965G(dev)) {
4837 for (i = 0; i < 16; i++)
4838 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4839 } else {
4840 for (i = 0; i < 8; i++)
4841 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4842 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4843 for (i = 0; i < 8; i++)
4844 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4845 }
Eric Anholt673a3942008-07-30 12:06:12 -07004846 i915_gem_detect_bit_6_swizzle(dev);
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05004847 init_waitqueue_head(&dev_priv->pending_flip_queue);
Eric Anholt673a3942008-07-30 12:06:12 -07004848}
Dave Airlie71acb5e2008-12-30 20:31:46 +10004849
4850/*
4851 * Create a physically contiguous memory object for this object
4852 * e.g. for cursor + overlay regs
4853 */
4854int i915_gem_init_phys_object(struct drm_device *dev,
4855 int id, int size)
4856{
4857 drm_i915_private_t *dev_priv = dev->dev_private;
4858 struct drm_i915_gem_phys_object *phys_obj;
4859 int ret;
4860
4861 if (dev_priv->mm.phys_objs[id - 1] || !size)
4862 return 0;
4863
Eric Anholt9a298b22009-03-24 12:23:04 -07004864 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004865 if (!phys_obj)
4866 return -ENOMEM;
4867
4868 phys_obj->id = id;
4869
Zhenyu Wange6be8d92010-01-05 11:25:05 +08004870 phys_obj->handle = drm_pci_alloc(dev, size, 0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004871 if (!phys_obj->handle) {
4872 ret = -ENOMEM;
4873 goto kfree_obj;
4874 }
4875#ifdef CONFIG_X86
4876 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4877#endif
4878
4879 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4880
4881 return 0;
4882kfree_obj:
Eric Anholt9a298b22009-03-24 12:23:04 -07004883 kfree(phys_obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004884 return ret;
4885}
4886
4887void i915_gem_free_phys_object(struct drm_device *dev, int id)
4888{
4889 drm_i915_private_t *dev_priv = dev->dev_private;
4890 struct drm_i915_gem_phys_object *phys_obj;
4891
4892 if (!dev_priv->mm.phys_objs[id - 1])
4893 return;
4894
4895 phys_obj = dev_priv->mm.phys_objs[id - 1];
4896 if (phys_obj->cur_obj) {
4897 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4898 }
4899
4900#ifdef CONFIG_X86
4901 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4902#endif
4903 drm_pci_free(dev, phys_obj->handle);
4904 kfree(phys_obj);
4905 dev_priv->mm.phys_objs[id - 1] = NULL;
4906}
4907
4908void i915_gem_free_all_phys_object(struct drm_device *dev)
4909{
4910 int i;
4911
Dave Airlie260883c2009-01-22 17:58:49 +10004912 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004913 i915_gem_free_phys_object(dev, i);
4914}
4915
4916void i915_gem_detach_phys_object(struct drm_device *dev,
4917 struct drm_gem_object *obj)
4918{
4919 struct drm_i915_gem_object *obj_priv;
4920 int i;
4921 int ret;
4922 int page_count;
4923
4924 obj_priv = obj->driver_private;
4925 if (!obj_priv->phys_obj)
4926 return;
4927
Chris Wilson4bdadb92010-01-27 13:36:32 +00004928 ret = i915_gem_object_get_pages(obj, 0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004929 if (ret)
4930 goto out;
4931
4932 page_count = obj->size / PAGE_SIZE;
4933
4934 for (i = 0; i < page_count; i++) {
Eric Anholt856fa192009-03-19 14:10:50 -07004935 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004936 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4937
4938 memcpy(dst, src, PAGE_SIZE);
4939 kunmap_atomic(dst, KM_USER0);
4940 }
Eric Anholt856fa192009-03-19 14:10:50 -07004941 drm_clflush_pages(obj_priv->pages, page_count);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004942 drm_agp_chipset_flush(dev);
Chris Wilsond78b47b2009-06-17 21:52:49 +01004943
4944 i915_gem_object_put_pages(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004945out:
4946 obj_priv->phys_obj->cur_obj = NULL;
4947 obj_priv->phys_obj = NULL;
4948}
4949
4950int
4951i915_gem_attach_phys_object(struct drm_device *dev,
4952 struct drm_gem_object *obj, int id)
4953{
4954 drm_i915_private_t *dev_priv = dev->dev_private;
4955 struct drm_i915_gem_object *obj_priv;
4956 int ret = 0;
4957 int page_count;
4958 int i;
4959
4960 if (id > I915_MAX_PHYS_OBJECT)
4961 return -EINVAL;
4962
4963 obj_priv = obj->driver_private;
4964
4965 if (obj_priv->phys_obj) {
4966 if (obj_priv->phys_obj->id == id)
4967 return 0;
4968 i915_gem_detach_phys_object(dev, obj);
4969 }
4970
4971
4972 /* create a new object */
4973 if (!dev_priv->mm.phys_objs[id - 1]) {
4974 ret = i915_gem_init_phys_object(dev, id,
4975 obj->size);
4976 if (ret) {
Linus Torvaldsaeb565d2009-01-26 10:01:53 -08004977 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004978 goto out;
4979 }
4980 }
4981
4982 /* bind to the object */
4983 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4984 obj_priv->phys_obj->cur_obj = obj;
4985
Chris Wilson4bdadb92010-01-27 13:36:32 +00004986 ret = i915_gem_object_get_pages(obj, 0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004987 if (ret) {
4988 DRM_ERROR("failed to get page list\n");
4989 goto out;
4990 }
4991
4992 page_count = obj->size / PAGE_SIZE;
4993
4994 for (i = 0; i < page_count; i++) {
Eric Anholt856fa192009-03-19 14:10:50 -07004995 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004996 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4997
4998 memcpy(dst, src, PAGE_SIZE);
4999 kunmap_atomic(src, KM_USER0);
5000 }
5001
Chris Wilsond78b47b2009-06-17 21:52:49 +01005002 i915_gem_object_put_pages(obj);
5003
Dave Airlie71acb5e2008-12-30 20:31:46 +10005004 return 0;
5005out:
5006 return ret;
5007}
5008
5009static int
5010i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
5011 struct drm_i915_gem_pwrite *args,
5012 struct drm_file *file_priv)
5013{
5014 struct drm_i915_gem_object *obj_priv = obj->driver_private;
5015 void *obj_addr;
5016 int ret;
5017 char __user *user_data;
5018
5019 user_data = (char __user *) (uintptr_t) args->data_ptr;
5020 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
5021
Zhao Yakui44d98a62009-10-09 11:39:40 +08005022 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
Dave Airlie71acb5e2008-12-30 20:31:46 +10005023 ret = copy_from_user(obj_addr, user_data, args->size);
5024 if (ret)
5025 return -EFAULT;
5026
5027 drm_agp_chipset_flush(dev);
5028 return 0;
5029}
Eric Anholtb9624422009-06-03 07:27:35 +00005030
5031void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
5032{
5033 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
5034
5035 /* Clean up our request list when the client is going away, so that
5036 * later retire_requests won't dereference our soon-to-be-gone
5037 * file_priv.
5038 */
5039 mutex_lock(&dev->struct_mutex);
5040 while (!list_empty(&i915_file_priv->mm.request_list))
5041 list_del_init(i915_file_priv->mm.request_list.next);
5042 mutex_unlock(&dev->struct_mutex);
5043}
Chris Wilson31169712009-09-14 16:50:28 +01005044
Chris Wilson31169712009-09-14 16:50:28 +01005045static int
5046i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask)
5047{
5048 drm_i915_private_t *dev_priv, *next_dev;
5049 struct drm_i915_gem_object *obj_priv, *next_obj;
5050 int cnt = 0;
5051 int would_deadlock = 1;
5052
5053 /* "fast-path" to count number of available objects */
5054 if (nr_to_scan == 0) {
5055 spin_lock(&shrink_list_lock);
5056 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5057 struct drm_device *dev = dev_priv->dev;
5058
5059 if (mutex_trylock(&dev->struct_mutex)) {
5060 list_for_each_entry(obj_priv,
5061 &dev_priv->mm.inactive_list,
5062 list)
5063 cnt++;
5064 mutex_unlock(&dev->struct_mutex);
5065 }
5066 }
5067 spin_unlock(&shrink_list_lock);
5068
5069 return (cnt / 100) * sysctl_vfs_cache_pressure;
5070 }
5071
5072 spin_lock(&shrink_list_lock);
5073
5074 /* first scan for clean buffers */
5075 list_for_each_entry_safe(dev_priv, next_dev,
5076 &shrink_list, mm.shrink_list) {
5077 struct drm_device *dev = dev_priv->dev;
5078
5079 if (! mutex_trylock(&dev->struct_mutex))
5080 continue;
5081
5082 spin_unlock(&shrink_list_lock);
5083
5084 i915_gem_retire_requests(dev);
5085
5086 list_for_each_entry_safe(obj_priv, next_obj,
5087 &dev_priv->mm.inactive_list,
5088 list) {
5089 if (i915_gem_object_is_purgeable(obj_priv)) {
Chris Wilson963b4832009-09-20 23:03:54 +01005090 i915_gem_object_unbind(obj_priv->obj);
Chris Wilson31169712009-09-14 16:50:28 +01005091 if (--nr_to_scan <= 0)
5092 break;
5093 }
5094 }
5095
5096 spin_lock(&shrink_list_lock);
5097 mutex_unlock(&dev->struct_mutex);
5098
Chris Wilson963b4832009-09-20 23:03:54 +01005099 would_deadlock = 0;
5100
Chris Wilson31169712009-09-14 16:50:28 +01005101 if (nr_to_scan <= 0)
5102 break;
5103 }
5104
5105 /* second pass, evict/count anything still on the inactive list */
5106 list_for_each_entry_safe(dev_priv, next_dev,
5107 &shrink_list, mm.shrink_list) {
5108 struct drm_device *dev = dev_priv->dev;
5109
5110 if (! mutex_trylock(&dev->struct_mutex))
5111 continue;
5112
5113 spin_unlock(&shrink_list_lock);
5114
5115 list_for_each_entry_safe(obj_priv, next_obj,
5116 &dev_priv->mm.inactive_list,
5117 list) {
5118 if (nr_to_scan > 0) {
Chris Wilson963b4832009-09-20 23:03:54 +01005119 i915_gem_object_unbind(obj_priv->obj);
Chris Wilson31169712009-09-14 16:50:28 +01005120 nr_to_scan--;
5121 } else
5122 cnt++;
5123 }
5124
5125 spin_lock(&shrink_list_lock);
5126 mutex_unlock(&dev->struct_mutex);
5127
5128 would_deadlock = 0;
5129 }
5130
5131 spin_unlock(&shrink_list_lock);
5132
5133 if (would_deadlock)
5134 return -1;
5135 else if (cnt > 0)
5136 return (cnt / 100) * sysctl_vfs_cache_pressure;
5137 else
5138 return 0;
5139}
5140
5141static struct shrinker shrinker = {
5142 .shrink = i915_gem_shrink,
5143 .seeks = DEFAULT_SEEKS,
5144};
5145
5146__init void
5147i915_gem_shrinker_init(void)
5148{
5149 register_shrinker(&shrinker);
5150}
5151
5152__exit void
5153i915_gem_shrinker_exit(void)
5154{
5155 unregister_shrinker(&shrinker);
5156}