blob: b8e3512744931e934722616644a207861a395a7c [file] [log] [blame]
Eric Anholt673a3942008-07-30 12:06:12 -07001/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
Chris Wilson1c5d22f2009-08-25 11:15:50 +010032#include "i915_trace.h"
Jesse Barnes652c3932009-08-17 13:31:43 -070033#include "intel_drv.h"
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090034#include <linux/slab.h>
Eric Anholt673a3942008-07-30 12:06:12 -070035#include <linux/swap.h>
Jesse Barnes79e53942008-11-07 14:24:08 -080036#include <linux/pci.h>
Eric Anholt673a3942008-07-30 12:06:12 -070037
Eric Anholte47c68e2008-11-14 13:35:19 -080038static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
39static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
40static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
Eric Anholte47c68e2008-11-14 13:35:19 -080041static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
42 int write);
43static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
44 uint64_t offset,
45 uint64_t size);
46static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
Eric Anholt673a3942008-07-30 12:06:12 -070047static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -080048static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
49 unsigned alignment);
Jesse Barnesde151cf2008-11-12 10:03:55 -080050static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
Chris Wilson07f73f62009-09-14 16:50:30 +010051static int i915_gem_evict_something(struct drm_device *dev, int min_size);
Chris Wilsonab5ee572009-09-20 19:25:47 +010052static int i915_gem_evict_from_inactive_list(struct drm_device *dev);
Dave Airlie71acb5e2008-12-30 20:31:46 +100053static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
54 struct drm_i915_gem_pwrite *args,
55 struct drm_file *file_priv);
Eric Anholt673a3942008-07-30 12:06:12 -070056
Chris Wilson31169712009-09-14 16:50:28 +010057static LIST_HEAD(shrink_list);
58static DEFINE_SPINLOCK(shrink_list_lock);
59
Jesse Barnes79e53942008-11-07 14:24:08 -080060int i915_gem_do_init(struct drm_device *dev, unsigned long start,
61 unsigned long end)
62{
63 drm_i915_private_t *dev_priv = dev->dev_private;
64
65 if (start >= end ||
66 (start & (PAGE_SIZE - 1)) != 0 ||
67 (end & (PAGE_SIZE - 1)) != 0) {
68 return -EINVAL;
69 }
70
71 drm_mm_init(&dev_priv->mm.gtt_space, start,
72 end - start);
73
74 dev->gtt_total = (uint32_t) (end - start);
75
76 return 0;
77}
Keith Packard6dbe2772008-10-14 21:41:13 -070078
Eric Anholt673a3942008-07-30 12:06:12 -070079int
80i915_gem_init_ioctl(struct drm_device *dev, void *data,
81 struct drm_file *file_priv)
82{
Eric Anholt673a3942008-07-30 12:06:12 -070083 struct drm_i915_gem_init *args = data;
Jesse Barnes79e53942008-11-07 14:24:08 -080084 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -070085
86 mutex_lock(&dev->struct_mutex);
Jesse Barnes79e53942008-11-07 14:24:08 -080087 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
Eric Anholt673a3942008-07-30 12:06:12 -070088 mutex_unlock(&dev->struct_mutex);
89
Jesse Barnes79e53942008-11-07 14:24:08 -080090 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -070091}
92
Eric Anholt5a125c32008-10-22 21:40:13 -070093int
94i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
95 struct drm_file *file_priv)
96{
Eric Anholt5a125c32008-10-22 21:40:13 -070097 struct drm_i915_gem_get_aperture *args = data;
Eric Anholt5a125c32008-10-22 21:40:13 -070098
99 if (!(dev->driver->driver_features & DRIVER_GEM))
100 return -ENODEV;
101
102 args->aper_size = dev->gtt_total;
Keith Packard2678d9d2008-11-20 22:54:54 -0800103 args->aper_available_size = (args->aper_size -
104 atomic_read(&dev->pin_memory));
Eric Anholt5a125c32008-10-22 21:40:13 -0700105
106 return 0;
107}
108
Eric Anholt673a3942008-07-30 12:06:12 -0700109
110/**
111 * Creates a new mm object and returns a handle to it.
112 */
113int
114i915_gem_create_ioctl(struct drm_device *dev, void *data,
115 struct drm_file *file_priv)
116{
117 struct drm_i915_gem_create *args = data;
118 struct drm_gem_object *obj;
Pekka Paalanena1a2d1d2009-08-23 12:40:55 +0300119 int ret;
120 u32 handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700121
122 args->size = roundup(args->size, PAGE_SIZE);
123
124 /* Allocate the new object */
Daniel Vetterac52bc52010-04-09 19:05:06 +0000125 obj = i915_gem_alloc_object(dev, args->size);
Eric Anholt673a3942008-07-30 12:06:12 -0700126 if (obj == NULL)
127 return -ENOMEM;
128
129 ret = drm_gem_handle_create(file_priv, obj, &handle);
Luca Barbieribc9025b2010-02-09 05:49:12 +0000130 drm_gem_object_handle_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700131
132 if (ret)
133 return ret;
134
135 args->handle = handle;
136
137 return 0;
138}
139
Eric Anholt40123c12009-03-09 13:42:30 -0700140static inline int
Eric Anholteb014592009-03-10 11:44:52 -0700141fast_shmem_read(struct page **pages,
142 loff_t page_base, int page_offset,
143 char __user *data,
144 int length)
145{
146 char __iomem *vaddr;
Florian Mickler2bc43b52009-04-06 22:55:41 +0200147 int unwritten;
Eric Anholteb014592009-03-10 11:44:52 -0700148
149 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
150 if (vaddr == NULL)
151 return -ENOMEM;
Florian Mickler2bc43b52009-04-06 22:55:41 +0200152 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
Eric Anholteb014592009-03-10 11:44:52 -0700153 kunmap_atomic(vaddr, KM_USER0);
154
Florian Mickler2bc43b52009-04-06 22:55:41 +0200155 if (unwritten)
156 return -EFAULT;
157
158 return 0;
Eric Anholteb014592009-03-10 11:44:52 -0700159}
160
Eric Anholt280b7132009-03-12 16:56:27 -0700161static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
162{
163 drm_i915_private_t *dev_priv = obj->dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +0100164 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt280b7132009-03-12 16:56:27 -0700165
166 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
167 obj_priv->tiling_mode != I915_TILING_NONE;
168}
169
Chris Wilson99a03df2010-05-27 14:15:34 +0100170static inline void
Eric Anholt40123c12009-03-09 13:42:30 -0700171slow_shmem_copy(struct page *dst_page,
172 int dst_offset,
173 struct page *src_page,
174 int src_offset,
175 int length)
176{
177 char *dst_vaddr, *src_vaddr;
178
Chris Wilson99a03df2010-05-27 14:15:34 +0100179 dst_vaddr = kmap(dst_page);
180 src_vaddr = kmap(src_page);
Eric Anholt40123c12009-03-09 13:42:30 -0700181
182 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
183
Chris Wilson99a03df2010-05-27 14:15:34 +0100184 kunmap(src_page);
185 kunmap(dst_page);
Eric Anholt40123c12009-03-09 13:42:30 -0700186}
187
Chris Wilson99a03df2010-05-27 14:15:34 +0100188static inline void
Eric Anholt280b7132009-03-12 16:56:27 -0700189slow_shmem_bit17_copy(struct page *gpu_page,
190 int gpu_offset,
191 struct page *cpu_page,
192 int cpu_offset,
193 int length,
194 int is_read)
195{
196 char *gpu_vaddr, *cpu_vaddr;
197
198 /* Use the unswizzled path if this page isn't affected. */
199 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
200 if (is_read)
201 return slow_shmem_copy(cpu_page, cpu_offset,
202 gpu_page, gpu_offset, length);
203 else
204 return slow_shmem_copy(gpu_page, gpu_offset,
205 cpu_page, cpu_offset, length);
206 }
207
Chris Wilson99a03df2010-05-27 14:15:34 +0100208 gpu_vaddr = kmap(gpu_page);
209 cpu_vaddr = kmap(cpu_page);
Eric Anholt280b7132009-03-12 16:56:27 -0700210
211 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
212 * XORing with the other bits (A9 for Y, A9 and A10 for X)
213 */
214 while (length > 0) {
215 int cacheline_end = ALIGN(gpu_offset + 1, 64);
216 int this_length = min(cacheline_end - gpu_offset, length);
217 int swizzled_gpu_offset = gpu_offset ^ 64;
218
219 if (is_read) {
220 memcpy(cpu_vaddr + cpu_offset,
221 gpu_vaddr + swizzled_gpu_offset,
222 this_length);
223 } else {
224 memcpy(gpu_vaddr + swizzled_gpu_offset,
225 cpu_vaddr + cpu_offset,
226 this_length);
227 }
228 cpu_offset += this_length;
229 gpu_offset += this_length;
230 length -= this_length;
231 }
232
Chris Wilson99a03df2010-05-27 14:15:34 +0100233 kunmap(cpu_page);
234 kunmap(gpu_page);
Eric Anholt280b7132009-03-12 16:56:27 -0700235}
236
Eric Anholt673a3942008-07-30 12:06:12 -0700237/**
Eric Anholteb014592009-03-10 11:44:52 -0700238 * This is the fast shmem pread path, which attempts to copy_from_user directly
239 * from the backing pages of the object to the user's address space. On a
240 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
241 */
242static int
243i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
244 struct drm_i915_gem_pread *args,
245 struct drm_file *file_priv)
246{
Daniel Vetter23010e42010-03-08 13:35:02 +0100247 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700248 ssize_t remain;
249 loff_t offset, page_base;
250 char __user *user_data;
251 int page_offset, page_length;
252 int ret;
253
254 user_data = (char __user *) (uintptr_t) args->data_ptr;
255 remain = args->size;
256
257 mutex_lock(&dev->struct_mutex);
258
Chris Wilson4bdadb92010-01-27 13:36:32 +0000259 ret = i915_gem_object_get_pages(obj, 0);
Eric Anholteb014592009-03-10 11:44:52 -0700260 if (ret != 0)
261 goto fail_unlock;
262
263 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
264 args->size);
265 if (ret != 0)
266 goto fail_put_pages;
267
Daniel Vetter23010e42010-03-08 13:35:02 +0100268 obj_priv = to_intel_bo(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700269 offset = args->offset;
270
271 while (remain > 0) {
272 /* Operation in this page
273 *
274 * page_base = page offset within aperture
275 * page_offset = offset within page
276 * page_length = bytes to copy for this page
277 */
278 page_base = (offset & ~(PAGE_SIZE-1));
279 page_offset = offset & (PAGE_SIZE-1);
280 page_length = remain;
281 if ((page_offset + remain) > PAGE_SIZE)
282 page_length = PAGE_SIZE - page_offset;
283
284 ret = fast_shmem_read(obj_priv->pages,
285 page_base, page_offset,
286 user_data, page_length);
287 if (ret)
288 goto fail_put_pages;
289
290 remain -= page_length;
291 user_data += page_length;
292 offset += page_length;
293 }
294
295fail_put_pages:
296 i915_gem_object_put_pages(obj);
297fail_unlock:
298 mutex_unlock(&dev->struct_mutex);
299
300 return ret;
301}
302
Chris Wilson07f73f62009-09-14 16:50:30 +0100303static int
304i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
305{
306 int ret;
307
Chris Wilson4bdadb92010-01-27 13:36:32 +0000308 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
Chris Wilson07f73f62009-09-14 16:50:30 +0100309
310 /* If we've insufficient memory to map in the pages, attempt
311 * to make some space by throwing out some old buffers.
312 */
313 if (ret == -ENOMEM) {
314 struct drm_device *dev = obj->dev;
Chris Wilson07f73f62009-09-14 16:50:30 +0100315
316 ret = i915_gem_evict_something(dev, obj->size);
317 if (ret)
318 return ret;
319
Chris Wilson4bdadb92010-01-27 13:36:32 +0000320 ret = i915_gem_object_get_pages(obj, 0);
Chris Wilson07f73f62009-09-14 16:50:30 +0100321 }
322
323 return ret;
324}
325
Eric Anholteb014592009-03-10 11:44:52 -0700326/**
327 * This is the fallback shmem pread path, which allocates temporary storage
328 * in kernel space to copy_to_user into outside of the struct_mutex, so we
329 * can copy out of the object's backing pages while holding the struct mutex
330 * and not take page faults.
331 */
332static int
333i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
334 struct drm_i915_gem_pread *args,
335 struct drm_file *file_priv)
336{
Daniel Vetter23010e42010-03-08 13:35:02 +0100337 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700338 struct mm_struct *mm = current->mm;
339 struct page **user_pages;
340 ssize_t remain;
341 loff_t offset, pinned_pages, i;
342 loff_t first_data_page, last_data_page, num_pages;
343 int shmem_page_index, shmem_page_offset;
344 int data_page_index, data_page_offset;
345 int page_length;
346 int ret;
347 uint64_t data_ptr = args->data_ptr;
Eric Anholt280b7132009-03-12 16:56:27 -0700348 int do_bit17_swizzling;
Eric Anholteb014592009-03-10 11:44:52 -0700349
350 remain = args->size;
351
352 /* Pin the user pages containing the data. We can't fault while
353 * holding the struct mutex, yet we want to hold it while
354 * dereferencing the user data.
355 */
356 first_data_page = data_ptr / PAGE_SIZE;
357 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
358 num_pages = last_data_page - first_data_page + 1;
359
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700360 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
Eric Anholteb014592009-03-10 11:44:52 -0700361 if (user_pages == NULL)
362 return -ENOMEM;
363
364 down_read(&mm->mmap_sem);
365 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
Eric Anholte5e9ecd2009-04-07 16:01:22 -0700366 num_pages, 1, 0, user_pages, NULL);
Eric Anholteb014592009-03-10 11:44:52 -0700367 up_read(&mm->mmap_sem);
368 if (pinned_pages < num_pages) {
369 ret = -EFAULT;
370 goto fail_put_user_pages;
371 }
372
Eric Anholt280b7132009-03-12 16:56:27 -0700373 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
374
Eric Anholteb014592009-03-10 11:44:52 -0700375 mutex_lock(&dev->struct_mutex);
376
Chris Wilson07f73f62009-09-14 16:50:30 +0100377 ret = i915_gem_object_get_pages_or_evict(obj);
378 if (ret)
Eric Anholteb014592009-03-10 11:44:52 -0700379 goto fail_unlock;
380
381 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
382 args->size);
383 if (ret != 0)
384 goto fail_put_pages;
385
Daniel Vetter23010e42010-03-08 13:35:02 +0100386 obj_priv = to_intel_bo(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700387 offset = args->offset;
388
389 while (remain > 0) {
390 /* Operation in this page
391 *
392 * shmem_page_index = page number within shmem file
393 * shmem_page_offset = offset within page in shmem file
394 * data_page_index = page number in get_user_pages return
395 * data_page_offset = offset with data_page_index page.
396 * page_length = bytes to copy for this page
397 */
398 shmem_page_index = offset / PAGE_SIZE;
399 shmem_page_offset = offset & ~PAGE_MASK;
400 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
401 data_page_offset = data_ptr & ~PAGE_MASK;
402
403 page_length = remain;
404 if ((shmem_page_offset + page_length) > PAGE_SIZE)
405 page_length = PAGE_SIZE - shmem_page_offset;
406 if ((data_page_offset + page_length) > PAGE_SIZE)
407 page_length = PAGE_SIZE - data_page_offset;
408
Eric Anholt280b7132009-03-12 16:56:27 -0700409 if (do_bit17_swizzling) {
Chris Wilson99a03df2010-05-27 14:15:34 +0100410 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
Eric Anholt280b7132009-03-12 16:56:27 -0700411 shmem_page_offset,
Chris Wilson99a03df2010-05-27 14:15:34 +0100412 user_pages[data_page_index],
413 data_page_offset,
414 page_length,
415 1);
416 } else {
417 slow_shmem_copy(user_pages[data_page_index],
418 data_page_offset,
419 obj_priv->pages[shmem_page_index],
420 shmem_page_offset,
421 page_length);
Eric Anholt280b7132009-03-12 16:56:27 -0700422 }
Eric Anholteb014592009-03-10 11:44:52 -0700423
424 remain -= page_length;
425 data_ptr += page_length;
426 offset += page_length;
427 }
428
429fail_put_pages:
430 i915_gem_object_put_pages(obj);
431fail_unlock:
432 mutex_unlock(&dev->struct_mutex);
433fail_put_user_pages:
434 for (i = 0; i < pinned_pages; i++) {
435 SetPageDirty(user_pages[i]);
436 page_cache_release(user_pages[i]);
437 }
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700438 drm_free_large(user_pages);
Eric Anholteb014592009-03-10 11:44:52 -0700439
440 return ret;
441}
442
Eric Anholt673a3942008-07-30 12:06:12 -0700443/**
444 * Reads data from the object referenced by handle.
445 *
446 * On error, the contents of *data are undefined.
447 */
448int
449i915_gem_pread_ioctl(struct drm_device *dev, void *data,
450 struct drm_file *file_priv)
451{
452 struct drm_i915_gem_pread *args = data;
453 struct drm_gem_object *obj;
454 struct drm_i915_gem_object *obj_priv;
Eric Anholt673a3942008-07-30 12:06:12 -0700455 int ret;
456
457 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
458 if (obj == NULL)
459 return -EBADF;
Daniel Vetter23010e42010-03-08 13:35:02 +0100460 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700461
462 /* Bounds check source.
463 *
464 * XXX: This could use review for overflow issues...
465 */
466 if (args->offset > obj->size || args->size > obj->size ||
467 args->offset + args->size > obj->size) {
Luca Barbieribc9025b2010-02-09 05:49:12 +0000468 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700469 return -EINVAL;
470 }
471
Eric Anholt280b7132009-03-12 16:56:27 -0700472 if (i915_gem_object_needs_bit17_swizzle(obj)) {
Eric Anholteb014592009-03-10 11:44:52 -0700473 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
Eric Anholt280b7132009-03-12 16:56:27 -0700474 } else {
475 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
476 if (ret != 0)
477 ret = i915_gem_shmem_pread_slow(dev, obj, args,
478 file_priv);
479 }
Eric Anholt673a3942008-07-30 12:06:12 -0700480
Luca Barbieribc9025b2010-02-09 05:49:12 +0000481 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700482
Eric Anholteb014592009-03-10 11:44:52 -0700483 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700484}
485
Keith Packard0839ccb2008-10-30 19:38:48 -0700486/* This is the fast write path which cannot handle
487 * page faults in the source data
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700488 */
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700489
Keith Packard0839ccb2008-10-30 19:38:48 -0700490static inline int
491fast_user_write(struct io_mapping *mapping,
492 loff_t page_base, int page_offset,
493 char __user *user_data,
494 int length)
495{
496 char *vaddr_atomic;
497 unsigned long unwritten;
498
499 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
500 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
501 user_data, length);
502 io_mapping_unmap_atomic(vaddr_atomic);
503 if (unwritten)
504 return -EFAULT;
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700505 return 0;
Keith Packard0839ccb2008-10-30 19:38:48 -0700506}
507
508/* Here's the write path which can sleep for
509 * page faults
510 */
511
512static inline int
Eric Anholt3de09aa2009-03-09 09:42:23 -0700513slow_kernel_write(struct io_mapping *mapping,
514 loff_t gtt_base, int gtt_offset,
515 struct page *user_page, int user_offset,
516 int length)
Keith Packard0839ccb2008-10-30 19:38:48 -0700517{
Eric Anholt3de09aa2009-03-09 09:42:23 -0700518 char *src_vaddr, *dst_vaddr;
Keith Packard0839ccb2008-10-30 19:38:48 -0700519 unsigned long unwritten;
520
Eric Anholt3de09aa2009-03-09 09:42:23 -0700521 dst_vaddr = io_mapping_map_atomic_wc(mapping, gtt_base);
522 src_vaddr = kmap_atomic(user_page, KM_USER1);
523 unwritten = __copy_from_user_inatomic_nocache(dst_vaddr + gtt_offset,
524 src_vaddr + user_offset,
525 length);
526 kunmap_atomic(src_vaddr, KM_USER1);
527 io_mapping_unmap_atomic(dst_vaddr);
Keith Packard0839ccb2008-10-30 19:38:48 -0700528 if (unwritten)
529 return -EFAULT;
530 return 0;
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700531}
532
Eric Anholt40123c12009-03-09 13:42:30 -0700533static inline int
534fast_shmem_write(struct page **pages,
535 loff_t page_base, int page_offset,
536 char __user *data,
537 int length)
538{
539 char __iomem *vaddr;
Dave Airlied0088772009-03-28 20:29:48 -0400540 unsigned long unwritten;
Eric Anholt40123c12009-03-09 13:42:30 -0700541
542 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
543 if (vaddr == NULL)
544 return -ENOMEM;
Dave Airlied0088772009-03-28 20:29:48 -0400545 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
Eric Anholt40123c12009-03-09 13:42:30 -0700546 kunmap_atomic(vaddr, KM_USER0);
547
Dave Airlied0088772009-03-28 20:29:48 -0400548 if (unwritten)
549 return -EFAULT;
Eric Anholt40123c12009-03-09 13:42:30 -0700550 return 0;
551}
552
Eric Anholt3de09aa2009-03-09 09:42:23 -0700553/**
554 * This is the fast pwrite path, where we copy the data directly from the
555 * user into the GTT, uncached.
556 */
Eric Anholt673a3942008-07-30 12:06:12 -0700557static int
Eric Anholt3de09aa2009-03-09 09:42:23 -0700558i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
559 struct drm_i915_gem_pwrite *args,
560 struct drm_file *file_priv)
Eric Anholt673a3942008-07-30 12:06:12 -0700561{
Daniel Vetter23010e42010-03-08 13:35:02 +0100562 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Keith Packard0839ccb2008-10-30 19:38:48 -0700563 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -0700564 ssize_t remain;
Keith Packard0839ccb2008-10-30 19:38:48 -0700565 loff_t offset, page_base;
Eric Anholt673a3942008-07-30 12:06:12 -0700566 char __user *user_data;
Keith Packard0839ccb2008-10-30 19:38:48 -0700567 int page_offset, page_length;
568 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700569
570 user_data = (char __user *) (uintptr_t) args->data_ptr;
571 remain = args->size;
572 if (!access_ok(VERIFY_READ, user_data, remain))
573 return -EFAULT;
574
575
576 mutex_lock(&dev->struct_mutex);
577 ret = i915_gem_object_pin(obj, 0);
578 if (ret) {
579 mutex_unlock(&dev->struct_mutex);
580 return ret;
581 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800582 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
Eric Anholt673a3942008-07-30 12:06:12 -0700583 if (ret)
584 goto fail;
585
Daniel Vetter23010e42010-03-08 13:35:02 +0100586 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700587 offset = obj_priv->gtt_offset + args->offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700588
589 while (remain > 0) {
590 /* Operation in this page
591 *
Keith Packard0839ccb2008-10-30 19:38:48 -0700592 * page_base = page offset within aperture
593 * page_offset = offset within page
594 * page_length = bytes to copy for this page
Eric Anholt673a3942008-07-30 12:06:12 -0700595 */
Keith Packard0839ccb2008-10-30 19:38:48 -0700596 page_base = (offset & ~(PAGE_SIZE-1));
597 page_offset = offset & (PAGE_SIZE-1);
598 page_length = remain;
599 if ((page_offset + remain) > PAGE_SIZE)
600 page_length = PAGE_SIZE - page_offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700601
Keith Packard0839ccb2008-10-30 19:38:48 -0700602 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
603 page_offset, user_data, page_length);
Eric Anholt673a3942008-07-30 12:06:12 -0700604
Keith Packard0839ccb2008-10-30 19:38:48 -0700605 /* If we get a fault while copying data, then (presumably) our
Eric Anholt3de09aa2009-03-09 09:42:23 -0700606 * source page isn't available. Return the error and we'll
607 * retry in the slow path.
Keith Packard0839ccb2008-10-30 19:38:48 -0700608 */
Eric Anholt3de09aa2009-03-09 09:42:23 -0700609 if (ret)
610 goto fail;
Eric Anholt673a3942008-07-30 12:06:12 -0700611
Keith Packard0839ccb2008-10-30 19:38:48 -0700612 remain -= page_length;
613 user_data += page_length;
614 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700615 }
Eric Anholt673a3942008-07-30 12:06:12 -0700616
617fail:
618 i915_gem_object_unpin(obj);
619 mutex_unlock(&dev->struct_mutex);
620
621 return ret;
622}
623
Eric Anholt3de09aa2009-03-09 09:42:23 -0700624/**
625 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
626 * the memory and maps it using kmap_atomic for copying.
627 *
628 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
629 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
630 */
Eric Anholt3043c602008-10-02 12:24:47 -0700631static int
Eric Anholt3de09aa2009-03-09 09:42:23 -0700632i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
633 struct drm_i915_gem_pwrite *args,
634 struct drm_file *file_priv)
Eric Anholt673a3942008-07-30 12:06:12 -0700635{
Daniel Vetter23010e42010-03-08 13:35:02 +0100636 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700637 drm_i915_private_t *dev_priv = dev->dev_private;
638 ssize_t remain;
639 loff_t gtt_page_base, offset;
640 loff_t first_data_page, last_data_page, num_pages;
641 loff_t pinned_pages, i;
642 struct page **user_pages;
643 struct mm_struct *mm = current->mm;
644 int gtt_page_offset, data_page_offset, data_page_index, page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700645 int ret;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700646 uint64_t data_ptr = args->data_ptr;
647
648 remain = args->size;
649
650 /* Pin the user pages containing the data. We can't fault while
651 * holding the struct mutex, and all of the pwrite implementations
652 * want to hold it while dereferencing the user data.
653 */
654 first_data_page = data_ptr / PAGE_SIZE;
655 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
656 num_pages = last_data_page - first_data_page + 1;
657
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700658 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
Eric Anholt3de09aa2009-03-09 09:42:23 -0700659 if (user_pages == NULL)
660 return -ENOMEM;
661
662 down_read(&mm->mmap_sem);
663 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
664 num_pages, 0, 0, user_pages, NULL);
665 up_read(&mm->mmap_sem);
666 if (pinned_pages < num_pages) {
667 ret = -EFAULT;
668 goto out_unpin_pages;
669 }
670
671 mutex_lock(&dev->struct_mutex);
672 ret = i915_gem_object_pin(obj, 0);
673 if (ret)
674 goto out_unlock;
675
676 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
677 if (ret)
678 goto out_unpin_object;
679
Daniel Vetter23010e42010-03-08 13:35:02 +0100680 obj_priv = to_intel_bo(obj);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700681 offset = obj_priv->gtt_offset + args->offset;
682
683 while (remain > 0) {
684 /* Operation in this page
685 *
686 * gtt_page_base = page offset within aperture
687 * gtt_page_offset = offset within page in aperture
688 * data_page_index = page number in get_user_pages return
689 * data_page_offset = offset with data_page_index page.
690 * page_length = bytes to copy for this page
691 */
692 gtt_page_base = offset & PAGE_MASK;
693 gtt_page_offset = offset & ~PAGE_MASK;
694 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
695 data_page_offset = data_ptr & ~PAGE_MASK;
696
697 page_length = remain;
698 if ((gtt_page_offset + page_length) > PAGE_SIZE)
699 page_length = PAGE_SIZE - gtt_page_offset;
700 if ((data_page_offset + page_length) > PAGE_SIZE)
701 page_length = PAGE_SIZE - data_page_offset;
702
703 ret = slow_kernel_write(dev_priv->mm.gtt_mapping,
704 gtt_page_base, gtt_page_offset,
705 user_pages[data_page_index],
706 data_page_offset,
707 page_length);
708
709 /* If we get a fault while copying data, then (presumably) our
710 * source page isn't available. Return the error and we'll
711 * retry in the slow path.
712 */
713 if (ret)
714 goto out_unpin_object;
715
716 remain -= page_length;
717 offset += page_length;
718 data_ptr += page_length;
719 }
720
721out_unpin_object:
722 i915_gem_object_unpin(obj);
723out_unlock:
724 mutex_unlock(&dev->struct_mutex);
725out_unpin_pages:
726 for (i = 0; i < pinned_pages; i++)
727 page_cache_release(user_pages[i]);
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700728 drm_free_large(user_pages);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700729
730 return ret;
731}
732
Eric Anholt40123c12009-03-09 13:42:30 -0700733/**
734 * This is the fast shmem pwrite path, which attempts to directly
735 * copy_from_user into the kmapped pages backing the object.
736 */
Eric Anholt673a3942008-07-30 12:06:12 -0700737static int
Eric Anholt40123c12009-03-09 13:42:30 -0700738i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
739 struct drm_i915_gem_pwrite *args,
740 struct drm_file *file_priv)
Eric Anholt673a3942008-07-30 12:06:12 -0700741{
Daniel Vetter23010e42010-03-08 13:35:02 +0100742 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt40123c12009-03-09 13:42:30 -0700743 ssize_t remain;
744 loff_t offset, page_base;
745 char __user *user_data;
746 int page_offset, page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700747 int ret;
Eric Anholt40123c12009-03-09 13:42:30 -0700748
749 user_data = (char __user *) (uintptr_t) args->data_ptr;
750 remain = args->size;
Eric Anholt673a3942008-07-30 12:06:12 -0700751
752 mutex_lock(&dev->struct_mutex);
753
Chris Wilson4bdadb92010-01-27 13:36:32 +0000754 ret = i915_gem_object_get_pages(obj, 0);
Eric Anholt40123c12009-03-09 13:42:30 -0700755 if (ret != 0)
756 goto fail_unlock;
757
Eric Anholte47c68e2008-11-14 13:35:19 -0800758 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
Eric Anholt40123c12009-03-09 13:42:30 -0700759 if (ret != 0)
760 goto fail_put_pages;
Eric Anholt673a3942008-07-30 12:06:12 -0700761
Daniel Vetter23010e42010-03-08 13:35:02 +0100762 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700763 offset = args->offset;
Eric Anholt40123c12009-03-09 13:42:30 -0700764 obj_priv->dirty = 1;
Eric Anholt673a3942008-07-30 12:06:12 -0700765
Eric Anholt40123c12009-03-09 13:42:30 -0700766 while (remain > 0) {
767 /* Operation in this page
768 *
769 * page_base = page offset within aperture
770 * page_offset = offset within page
771 * page_length = bytes to copy for this page
772 */
773 page_base = (offset & ~(PAGE_SIZE-1));
774 page_offset = offset & (PAGE_SIZE-1);
775 page_length = remain;
776 if ((page_offset + remain) > PAGE_SIZE)
777 page_length = PAGE_SIZE - page_offset;
778
779 ret = fast_shmem_write(obj_priv->pages,
780 page_base, page_offset,
781 user_data, page_length);
782 if (ret)
783 goto fail_put_pages;
784
785 remain -= page_length;
786 user_data += page_length;
787 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700788 }
789
Eric Anholt40123c12009-03-09 13:42:30 -0700790fail_put_pages:
791 i915_gem_object_put_pages(obj);
792fail_unlock:
Eric Anholt673a3942008-07-30 12:06:12 -0700793 mutex_unlock(&dev->struct_mutex);
794
Eric Anholt40123c12009-03-09 13:42:30 -0700795 return ret;
796}
797
798/**
799 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
800 * the memory and maps it using kmap_atomic for copying.
801 *
802 * This avoids taking mmap_sem for faulting on the user's address while the
803 * struct_mutex is held.
804 */
805static int
806i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
807 struct drm_i915_gem_pwrite *args,
808 struct drm_file *file_priv)
809{
Daniel Vetter23010e42010-03-08 13:35:02 +0100810 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt40123c12009-03-09 13:42:30 -0700811 struct mm_struct *mm = current->mm;
812 struct page **user_pages;
813 ssize_t remain;
814 loff_t offset, pinned_pages, i;
815 loff_t first_data_page, last_data_page, num_pages;
816 int shmem_page_index, shmem_page_offset;
817 int data_page_index, data_page_offset;
818 int page_length;
819 int ret;
820 uint64_t data_ptr = args->data_ptr;
Eric Anholt280b7132009-03-12 16:56:27 -0700821 int do_bit17_swizzling;
Eric Anholt40123c12009-03-09 13:42:30 -0700822
823 remain = args->size;
824
825 /* Pin the user pages containing the data. We can't fault while
826 * holding the struct mutex, and all of the pwrite implementations
827 * want to hold it while dereferencing the user data.
828 */
829 first_data_page = data_ptr / PAGE_SIZE;
830 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
831 num_pages = last_data_page - first_data_page + 1;
832
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700833 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
Eric Anholt40123c12009-03-09 13:42:30 -0700834 if (user_pages == NULL)
835 return -ENOMEM;
836
837 down_read(&mm->mmap_sem);
838 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
839 num_pages, 0, 0, user_pages, NULL);
840 up_read(&mm->mmap_sem);
841 if (pinned_pages < num_pages) {
842 ret = -EFAULT;
843 goto fail_put_user_pages;
844 }
845
Eric Anholt280b7132009-03-12 16:56:27 -0700846 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
847
Eric Anholt40123c12009-03-09 13:42:30 -0700848 mutex_lock(&dev->struct_mutex);
849
Chris Wilson07f73f62009-09-14 16:50:30 +0100850 ret = i915_gem_object_get_pages_or_evict(obj);
851 if (ret)
Eric Anholt40123c12009-03-09 13:42:30 -0700852 goto fail_unlock;
853
854 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
855 if (ret != 0)
856 goto fail_put_pages;
857
Daniel Vetter23010e42010-03-08 13:35:02 +0100858 obj_priv = to_intel_bo(obj);
Eric Anholt40123c12009-03-09 13:42:30 -0700859 offset = args->offset;
860 obj_priv->dirty = 1;
861
862 while (remain > 0) {
863 /* Operation in this page
864 *
865 * shmem_page_index = page number within shmem file
866 * shmem_page_offset = offset within page in shmem file
867 * data_page_index = page number in get_user_pages return
868 * data_page_offset = offset with data_page_index page.
869 * page_length = bytes to copy for this page
870 */
871 shmem_page_index = offset / PAGE_SIZE;
872 shmem_page_offset = offset & ~PAGE_MASK;
873 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
874 data_page_offset = data_ptr & ~PAGE_MASK;
875
876 page_length = remain;
877 if ((shmem_page_offset + page_length) > PAGE_SIZE)
878 page_length = PAGE_SIZE - shmem_page_offset;
879 if ((data_page_offset + page_length) > PAGE_SIZE)
880 page_length = PAGE_SIZE - data_page_offset;
881
Eric Anholt280b7132009-03-12 16:56:27 -0700882 if (do_bit17_swizzling) {
Chris Wilson99a03df2010-05-27 14:15:34 +0100883 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
Eric Anholt280b7132009-03-12 16:56:27 -0700884 shmem_page_offset,
885 user_pages[data_page_index],
886 data_page_offset,
Chris Wilson99a03df2010-05-27 14:15:34 +0100887 page_length,
888 0);
889 } else {
890 slow_shmem_copy(obj_priv->pages[shmem_page_index],
891 shmem_page_offset,
892 user_pages[data_page_index],
893 data_page_offset,
894 page_length);
Eric Anholt280b7132009-03-12 16:56:27 -0700895 }
Eric Anholt40123c12009-03-09 13:42:30 -0700896
897 remain -= page_length;
898 data_ptr += page_length;
899 offset += page_length;
900 }
901
902fail_put_pages:
903 i915_gem_object_put_pages(obj);
904fail_unlock:
905 mutex_unlock(&dev->struct_mutex);
906fail_put_user_pages:
907 for (i = 0; i < pinned_pages; i++)
908 page_cache_release(user_pages[i]);
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700909 drm_free_large(user_pages);
Eric Anholt40123c12009-03-09 13:42:30 -0700910
911 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700912}
913
914/**
915 * Writes data to the object referenced by handle.
916 *
917 * On error, the contents of the buffer that were to be modified are undefined.
918 */
919int
920i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
921 struct drm_file *file_priv)
922{
923 struct drm_i915_gem_pwrite *args = data;
924 struct drm_gem_object *obj;
925 struct drm_i915_gem_object *obj_priv;
926 int ret = 0;
927
928 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
929 if (obj == NULL)
930 return -EBADF;
Daniel Vetter23010e42010-03-08 13:35:02 +0100931 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700932
933 /* Bounds check destination.
934 *
935 * XXX: This could use review for overflow issues...
936 */
937 if (args->offset > obj->size || args->size > obj->size ||
938 args->offset + args->size > obj->size) {
Luca Barbieribc9025b2010-02-09 05:49:12 +0000939 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700940 return -EINVAL;
941 }
942
943 /* We can only do the GTT pwrite on untiled buffers, as otherwise
944 * it would end up going through the fenced access, and we'll get
945 * different detiling behavior between reading and writing.
946 * pread/pwrite currently are reading and writing from the CPU
947 * perspective, requiring manual detiling by the client.
948 */
Dave Airlie71acb5e2008-12-30 20:31:46 +1000949 if (obj_priv->phys_obj)
950 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
951 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
Chris Wilson9b8c4a02010-05-27 14:21:01 +0100952 dev->gtt_total != 0 &&
953 obj->write_domain != I915_GEM_DOMAIN_CPU) {
Eric Anholt3de09aa2009-03-09 09:42:23 -0700954 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
955 if (ret == -EFAULT) {
956 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
957 file_priv);
958 }
Eric Anholt280b7132009-03-12 16:56:27 -0700959 } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
960 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
Eric Anholt40123c12009-03-09 13:42:30 -0700961 } else {
962 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
963 if (ret == -EFAULT) {
964 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
965 file_priv);
966 }
967 }
Eric Anholt673a3942008-07-30 12:06:12 -0700968
969#if WATCH_PWRITE
970 if (ret)
971 DRM_INFO("pwrite failed %d\n", ret);
972#endif
973
Luca Barbieribc9025b2010-02-09 05:49:12 +0000974 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700975
976 return ret;
977}
978
979/**
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800980 * Called when user space prepares to use an object with the CPU, either
981 * through the mmap ioctl's mapping or a GTT mapping.
Eric Anholt673a3942008-07-30 12:06:12 -0700982 */
983int
984i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
985 struct drm_file *file_priv)
986{
Eric Anholta09ba7f2009-08-29 12:49:51 -0700987 struct drm_i915_private *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -0700988 struct drm_i915_gem_set_domain *args = data;
989 struct drm_gem_object *obj;
Jesse Barnes652c3932009-08-17 13:31:43 -0700990 struct drm_i915_gem_object *obj_priv;
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800991 uint32_t read_domains = args->read_domains;
992 uint32_t write_domain = args->write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -0700993 int ret;
994
995 if (!(dev->driver->driver_features & DRIVER_GEM))
996 return -ENODEV;
997
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800998 /* Only handle setting domains to types used by the CPU. */
Chris Wilson21d509e2009-06-06 09:46:02 +0100999 if (write_domain & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001000 return -EINVAL;
1001
Chris Wilson21d509e2009-06-06 09:46:02 +01001002 if (read_domains & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001003 return -EINVAL;
1004
1005 /* Having something in the write domain implies it's in the read
1006 * domain, and only that read domain. Enforce that in the request.
1007 */
1008 if (write_domain != 0 && read_domains != write_domain)
1009 return -EINVAL;
1010
Eric Anholt673a3942008-07-30 12:06:12 -07001011 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1012 if (obj == NULL)
1013 return -EBADF;
Daniel Vetter23010e42010-03-08 13:35:02 +01001014 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001015
1016 mutex_lock(&dev->struct_mutex);
Jesse Barnes652c3932009-08-17 13:31:43 -07001017
1018 intel_mark_busy(dev, obj);
1019
Eric Anholt673a3942008-07-30 12:06:12 -07001020#if WATCH_BUF
Krzysztof Halasacfd43c02009-06-20 00:31:28 +02001021 DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001022 obj, obj->size, read_domains, write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07001023#endif
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001024 if (read_domains & I915_GEM_DOMAIN_GTT) {
1025 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
Eric Anholt02354392008-11-26 13:58:13 -08001026
Eric Anholta09ba7f2009-08-29 12:49:51 -07001027 /* Update the LRU on the fence for the CPU access that's
1028 * about to occur.
1029 */
1030 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
Daniel Vetter007cc8a2010-04-28 11:02:31 +02001031 struct drm_i915_fence_reg *reg =
1032 &dev_priv->fence_regs[obj_priv->fence_reg];
1033 list_move_tail(&reg->lru_list,
Eric Anholta09ba7f2009-08-29 12:49:51 -07001034 &dev_priv->mm.fence_list);
1035 }
1036
Eric Anholt02354392008-11-26 13:58:13 -08001037 /* Silently promote "you're not bound, there was nothing to do"
1038 * to success, since the client was just asking us to
1039 * make sure everything was done.
1040 */
1041 if (ret == -EINVAL)
1042 ret = 0;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001043 } else {
Eric Anholte47c68e2008-11-14 13:35:19 -08001044 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001045 }
1046
Eric Anholt673a3942008-07-30 12:06:12 -07001047 drm_gem_object_unreference(obj);
1048 mutex_unlock(&dev->struct_mutex);
1049 return ret;
1050}
1051
1052/**
1053 * Called when user space has done writes to this buffer
1054 */
1055int
1056i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1057 struct drm_file *file_priv)
1058{
1059 struct drm_i915_gem_sw_finish *args = data;
1060 struct drm_gem_object *obj;
1061 struct drm_i915_gem_object *obj_priv;
1062 int ret = 0;
1063
1064 if (!(dev->driver->driver_features & DRIVER_GEM))
1065 return -ENODEV;
1066
1067 mutex_lock(&dev->struct_mutex);
1068 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1069 if (obj == NULL) {
1070 mutex_unlock(&dev->struct_mutex);
1071 return -EBADF;
1072 }
1073
1074#if WATCH_BUF
Krzysztof Halasacfd43c02009-06-20 00:31:28 +02001075 DRM_INFO("%s: sw_finish %d (%p %zd)\n",
Eric Anholt673a3942008-07-30 12:06:12 -07001076 __func__, args->handle, obj, obj->size);
1077#endif
Daniel Vetter23010e42010-03-08 13:35:02 +01001078 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001079
1080 /* Pinned buffers may be scanout, so flush the cache */
Eric Anholte47c68e2008-11-14 13:35:19 -08001081 if (obj_priv->pin_count)
1082 i915_gem_object_flush_cpu_write_domain(obj);
1083
Eric Anholt673a3942008-07-30 12:06:12 -07001084 drm_gem_object_unreference(obj);
1085 mutex_unlock(&dev->struct_mutex);
1086 return ret;
1087}
1088
1089/**
1090 * Maps the contents of an object, returning the address it is mapped
1091 * into.
1092 *
1093 * While the mapping holds a reference on the contents of the object, it doesn't
1094 * imply a ref on the object itself.
1095 */
1096int
1097i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1098 struct drm_file *file_priv)
1099{
1100 struct drm_i915_gem_mmap *args = data;
1101 struct drm_gem_object *obj;
1102 loff_t offset;
1103 unsigned long addr;
1104
1105 if (!(dev->driver->driver_features & DRIVER_GEM))
1106 return -ENODEV;
1107
1108 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1109 if (obj == NULL)
1110 return -EBADF;
1111
1112 offset = args->offset;
1113
1114 down_write(&current->mm->mmap_sem);
1115 addr = do_mmap(obj->filp, 0, args->size,
1116 PROT_READ | PROT_WRITE, MAP_SHARED,
1117 args->offset);
1118 up_write(&current->mm->mmap_sem);
Luca Barbieribc9025b2010-02-09 05:49:12 +00001119 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001120 if (IS_ERR((void *)addr))
1121 return addr;
1122
1123 args->addr_ptr = (uint64_t) addr;
1124
1125 return 0;
1126}
1127
Jesse Barnesde151cf2008-11-12 10:03:55 -08001128/**
1129 * i915_gem_fault - fault a page into the GTT
1130 * vma: VMA in question
1131 * vmf: fault info
1132 *
1133 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1134 * from userspace. The fault handler takes care of binding the object to
1135 * the GTT (if needed), allocating and programming a fence register (again,
1136 * only if needed based on whether the old reg is still valid or the object
1137 * is tiled) and inserting a new PTE into the faulting process.
1138 *
1139 * Note that the faulting process may involve evicting existing objects
1140 * from the GTT and/or fence registers to make room. So performance may
1141 * suffer if the GTT working set is large or there are few fence registers
1142 * left.
1143 */
1144int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1145{
1146 struct drm_gem_object *obj = vma->vm_private_data;
1147 struct drm_device *dev = obj->dev;
1148 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001149 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001150 pgoff_t page_offset;
1151 unsigned long pfn;
1152 int ret = 0;
Jesse Barnes0f973f22009-01-26 17:10:45 -08001153 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001154
1155 /* We don't use vmf->pgoff since that has the fake offset */
1156 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1157 PAGE_SHIFT;
1158
1159 /* Now bind it into the GTT if needed */
1160 mutex_lock(&dev->struct_mutex);
1161 if (!obj_priv->gtt_space) {
Chris Wilsone67b8ce2009-09-14 16:50:26 +01001162 ret = i915_gem_object_bind_to_gtt(obj, 0);
Chris Wilsonc7150892009-09-23 00:43:56 +01001163 if (ret)
1164 goto unlock;
Kristian Høgsberg07f4f3e2009-05-27 14:37:28 -04001165
Jesse Barnes14b60392009-05-20 16:47:08 -04001166 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001167
1168 ret = i915_gem_object_set_to_gtt_domain(obj, write);
Chris Wilsonc7150892009-09-23 00:43:56 +01001169 if (ret)
1170 goto unlock;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001171 }
1172
1173 /* Need a new fence register? */
Eric Anholta09ba7f2009-08-29 12:49:51 -07001174 if (obj_priv->tiling_mode != I915_TILING_NONE) {
Chris Wilson8c4b8c32009-06-17 22:08:52 +01001175 ret = i915_gem_object_get_fence_reg(obj);
Chris Wilsonc7150892009-09-23 00:43:56 +01001176 if (ret)
1177 goto unlock;
Eric Anholtd9ddcb92009-01-27 10:33:49 -08001178 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08001179
1180 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1181 page_offset;
1182
1183 /* Finally, remap it using the new GTT offset */
1184 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
Chris Wilsonc7150892009-09-23 00:43:56 +01001185unlock:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001186 mutex_unlock(&dev->struct_mutex);
1187
1188 switch (ret) {
Chris Wilsonc7150892009-09-23 00:43:56 +01001189 case 0:
1190 case -ERESTARTSYS:
1191 return VM_FAULT_NOPAGE;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001192 case -ENOMEM:
1193 case -EAGAIN:
1194 return VM_FAULT_OOM;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001195 default:
Chris Wilsonc7150892009-09-23 00:43:56 +01001196 return VM_FAULT_SIGBUS;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001197 }
1198}
1199
1200/**
1201 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1202 * @obj: obj in question
1203 *
1204 * GEM memory mapping works by handing back to userspace a fake mmap offset
1205 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1206 * up the object based on the offset and sets up the various memory mapping
1207 * structures.
1208 *
1209 * This routine allocates and attaches a fake offset for @obj.
1210 */
1211static int
1212i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1213{
1214 struct drm_device *dev = obj->dev;
1215 struct drm_gem_mm *mm = dev->mm_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001216 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001217 struct drm_map_list *list;
Benjamin Herrenschmidtf77d3902009-02-02 16:55:46 +11001218 struct drm_local_map *map;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001219 int ret = 0;
1220
1221 /* Set the object up for mmap'ing */
1222 list = &obj->map_list;
Eric Anholt9a298b22009-03-24 12:23:04 -07001223 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001224 if (!list->map)
1225 return -ENOMEM;
1226
1227 map = list->map;
1228 map->type = _DRM_GEM;
1229 map->size = obj->size;
1230 map->handle = obj;
1231
1232 /* Get a DRM GEM mmap offset allocated... */
1233 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1234 obj->size / PAGE_SIZE, 0, 0);
1235 if (!list->file_offset_node) {
1236 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
1237 ret = -ENOMEM;
1238 goto out_free_list;
1239 }
1240
1241 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1242 obj->size / PAGE_SIZE, 0);
1243 if (!list->file_offset_node) {
1244 ret = -ENOMEM;
1245 goto out_free_list;
1246 }
1247
1248 list->hash.key = list->file_offset_node->start;
1249 if (drm_ht_insert_item(&mm->offset_hash, &list->hash)) {
1250 DRM_ERROR("failed to add to map hash\n");
Chris Wilson5618ca62009-12-02 15:15:30 +00001251 ret = -ENOMEM;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001252 goto out_free_mm;
1253 }
1254
1255 /* By now we should be all set, any drm_mmap request on the offset
1256 * below will get to our mmap & fault handler */
1257 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1258
1259 return 0;
1260
1261out_free_mm:
1262 drm_mm_put_block(list->file_offset_node);
1263out_free_list:
Eric Anholt9a298b22009-03-24 12:23:04 -07001264 kfree(list->map);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001265
1266 return ret;
1267}
1268
Chris Wilson901782b2009-07-10 08:18:50 +01001269/**
1270 * i915_gem_release_mmap - remove physical page mappings
1271 * @obj: obj in question
1272 *
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02001273 * Preserve the reservation of the mmapping with the DRM core code, but
Chris Wilson901782b2009-07-10 08:18:50 +01001274 * relinquish ownership of the pages back to the system.
1275 *
1276 * It is vital that we remove the page mapping if we have mapped a tiled
1277 * object through the GTT and then lose the fence register due to
1278 * resource pressure. Similarly if the object has been moved out of the
1279 * aperture, than pages mapped into userspace must be revoked. Removing the
1280 * mapping will then trigger a page fault on the next user access, allowing
1281 * fixup by i915_gem_fault().
1282 */
Eric Anholtd05ca302009-07-10 13:02:26 -07001283void
Chris Wilson901782b2009-07-10 08:18:50 +01001284i915_gem_release_mmap(struct drm_gem_object *obj)
1285{
1286 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001287 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson901782b2009-07-10 08:18:50 +01001288
1289 if (dev->dev_mapping)
1290 unmap_mapping_range(dev->dev_mapping,
1291 obj_priv->mmap_offset, obj->size, 1);
1292}
1293
Jesse Barnesab00b3e2009-02-11 14:01:46 -08001294static void
1295i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1296{
1297 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001298 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesab00b3e2009-02-11 14:01:46 -08001299 struct drm_gem_mm *mm = dev->mm_private;
1300 struct drm_map_list *list;
1301
1302 list = &obj->map_list;
1303 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1304
1305 if (list->file_offset_node) {
1306 drm_mm_put_block(list->file_offset_node);
1307 list->file_offset_node = NULL;
1308 }
1309
1310 if (list->map) {
Eric Anholt9a298b22009-03-24 12:23:04 -07001311 kfree(list->map);
Jesse Barnesab00b3e2009-02-11 14:01:46 -08001312 list->map = NULL;
1313 }
1314
1315 obj_priv->mmap_offset = 0;
1316}
1317
Jesse Barnesde151cf2008-11-12 10:03:55 -08001318/**
1319 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1320 * @obj: object to check
1321 *
1322 * Return the required GTT alignment for an object, taking into account
1323 * potential fence register mapping if needed.
1324 */
1325static uint32_t
1326i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1327{
1328 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001329 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001330 int start, i;
1331
1332 /*
1333 * Minimum alignment is 4k (GTT page size), but might be greater
1334 * if a fence register is needed for the object.
1335 */
1336 if (IS_I965G(dev) || obj_priv->tiling_mode == I915_TILING_NONE)
1337 return 4096;
1338
1339 /*
1340 * Previous chips need to be aligned to the size of the smallest
1341 * fence register that can contain the object.
1342 */
1343 if (IS_I9XX(dev))
1344 start = 1024*1024;
1345 else
1346 start = 512*1024;
1347
1348 for (i = start; i < obj->size; i <<= 1)
1349 ;
1350
1351 return i;
1352}
1353
1354/**
1355 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1356 * @dev: DRM device
1357 * @data: GTT mapping ioctl data
1358 * @file_priv: GEM object info
1359 *
1360 * Simply returns the fake offset to userspace so it can mmap it.
1361 * The mmap call will end up in drm_gem_mmap(), which will set things
1362 * up so we can get faults in the handler above.
1363 *
1364 * The fault handler will take care of binding the object into the GTT
1365 * (since it may have been evicted to make room for something), allocating
1366 * a fence register, and mapping the appropriate aperture address into
1367 * userspace.
1368 */
1369int
1370i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1371 struct drm_file *file_priv)
1372{
1373 struct drm_i915_gem_mmap_gtt *args = data;
1374 struct drm_i915_private *dev_priv = dev->dev_private;
1375 struct drm_gem_object *obj;
1376 struct drm_i915_gem_object *obj_priv;
1377 int ret;
1378
1379 if (!(dev->driver->driver_features & DRIVER_GEM))
1380 return -ENODEV;
1381
1382 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1383 if (obj == NULL)
1384 return -EBADF;
1385
1386 mutex_lock(&dev->struct_mutex);
1387
Daniel Vetter23010e42010-03-08 13:35:02 +01001388 obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001389
Chris Wilsonab182822009-09-22 18:46:17 +01001390 if (obj_priv->madv != I915_MADV_WILLNEED) {
1391 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1392 drm_gem_object_unreference(obj);
1393 mutex_unlock(&dev->struct_mutex);
1394 return -EINVAL;
1395 }
1396
1397
Jesse Barnesde151cf2008-11-12 10:03:55 -08001398 if (!obj_priv->mmap_offset) {
1399 ret = i915_gem_create_mmap_offset(obj);
Chris Wilson13af1062009-02-11 14:26:31 +00001400 if (ret) {
1401 drm_gem_object_unreference(obj);
1402 mutex_unlock(&dev->struct_mutex);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001403 return ret;
Chris Wilson13af1062009-02-11 14:26:31 +00001404 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08001405 }
1406
1407 args->offset = obj_priv->mmap_offset;
1408
Jesse Barnesde151cf2008-11-12 10:03:55 -08001409 /*
1410 * Pull it into the GTT so that we have a page list (makes the
1411 * initial fault faster and any subsequent flushing possible).
1412 */
1413 if (!obj_priv->agp_mem) {
Chris Wilsone67b8ce2009-09-14 16:50:26 +01001414 ret = i915_gem_object_bind_to_gtt(obj, 0);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001415 if (ret) {
1416 drm_gem_object_unreference(obj);
1417 mutex_unlock(&dev->struct_mutex);
1418 return ret;
1419 }
Jesse Barnes14b60392009-05-20 16:47:08 -04001420 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001421 }
1422
1423 drm_gem_object_unreference(obj);
1424 mutex_unlock(&dev->struct_mutex);
1425
1426 return 0;
1427}
1428
Ben Gamari6911a9b2009-04-02 11:24:54 -07001429void
Eric Anholt856fa192009-03-19 14:10:50 -07001430i915_gem_object_put_pages(struct drm_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07001431{
Daniel Vetter23010e42010-03-08 13:35:02 +01001432 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001433 int page_count = obj->size / PAGE_SIZE;
1434 int i;
1435
Eric Anholt856fa192009-03-19 14:10:50 -07001436 BUG_ON(obj_priv->pages_refcount == 0);
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001437 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
Eric Anholt856fa192009-03-19 14:10:50 -07001438
1439 if (--obj_priv->pages_refcount != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07001440 return;
1441
Eric Anholt280b7132009-03-12 16:56:27 -07001442 if (obj_priv->tiling_mode != I915_TILING_NONE)
1443 i915_gem_object_save_bit_17_swizzle(obj);
1444
Chris Wilson3ef94da2009-09-14 16:50:29 +01001445 if (obj_priv->madv == I915_MADV_DONTNEED)
Chris Wilson13a05fd2009-09-20 23:03:19 +01001446 obj_priv->dirty = 0;
Chris Wilson3ef94da2009-09-14 16:50:29 +01001447
1448 for (i = 0; i < page_count; i++) {
Chris Wilson3ef94da2009-09-14 16:50:29 +01001449 if (obj_priv->dirty)
1450 set_page_dirty(obj_priv->pages[i]);
1451
1452 if (obj_priv->madv == I915_MADV_WILLNEED)
Eric Anholt856fa192009-03-19 14:10:50 -07001453 mark_page_accessed(obj_priv->pages[i]);
Chris Wilson3ef94da2009-09-14 16:50:29 +01001454
1455 page_cache_release(obj_priv->pages[i]);
1456 }
Eric Anholt673a3942008-07-30 12:06:12 -07001457 obj_priv->dirty = 0;
1458
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07001459 drm_free_large(obj_priv->pages);
Eric Anholt856fa192009-03-19 14:10:50 -07001460 obj_priv->pages = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07001461}
1462
1463static void
Zou Nan hai852835f2010-05-21 09:08:56 +08001464i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno,
1465 struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001466{
1467 struct drm_device *dev = obj->dev;
1468 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001469 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Zou Nan hai852835f2010-05-21 09:08:56 +08001470 BUG_ON(ring == NULL);
1471 obj_priv->ring = ring;
Eric Anholt673a3942008-07-30 12:06:12 -07001472
1473 /* Add a reference if we're newly entering the active list. */
1474 if (!obj_priv->active) {
1475 drm_gem_object_reference(obj);
1476 obj_priv->active = 1;
1477 }
1478 /* Move from whatever list we were on to the tail of execution. */
Carl Worth5e118f42009-03-20 11:54:25 -07001479 spin_lock(&dev_priv->mm.active_list_lock);
Zou Nan hai852835f2010-05-21 09:08:56 +08001480 list_move_tail(&obj_priv->list, &ring->active_list);
Carl Worth5e118f42009-03-20 11:54:25 -07001481 spin_unlock(&dev_priv->mm.active_list_lock);
Eric Anholtce44b0e2008-11-06 16:00:31 -08001482 obj_priv->last_rendering_seqno = seqno;
Eric Anholt673a3942008-07-30 12:06:12 -07001483}
1484
Eric Anholtce44b0e2008-11-06 16:00:31 -08001485static void
1486i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1487{
1488 struct drm_device *dev = obj->dev;
1489 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001490 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholtce44b0e2008-11-06 16:00:31 -08001491
1492 BUG_ON(!obj_priv->active);
1493 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1494 obj_priv->last_rendering_seqno = 0;
1495}
Eric Anholt673a3942008-07-30 12:06:12 -07001496
Chris Wilson963b4832009-09-20 23:03:54 +01001497/* Immediately discard the backing storage */
1498static void
1499i915_gem_object_truncate(struct drm_gem_object *obj)
1500{
Daniel Vetter23010e42010-03-08 13:35:02 +01001501 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001502 struct inode *inode;
Chris Wilson963b4832009-09-20 23:03:54 +01001503
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001504 inode = obj->filp->f_path.dentry->d_inode;
1505 if (inode->i_op->truncate)
1506 inode->i_op->truncate (inode);
1507
1508 obj_priv->madv = __I915_MADV_PURGED;
Chris Wilson963b4832009-09-20 23:03:54 +01001509}
1510
1511static inline int
1512i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1513{
1514 return obj_priv->madv == I915_MADV_DONTNEED;
1515}
1516
Eric Anholt673a3942008-07-30 12:06:12 -07001517static void
1518i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1519{
1520 struct drm_device *dev = obj->dev;
1521 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001522 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001523
1524 i915_verify_inactive(dev, __FILE__, __LINE__);
1525 if (obj_priv->pin_count != 0)
1526 list_del_init(&obj_priv->list);
1527 else
1528 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1529
Daniel Vetter99fcb762010-02-07 16:20:18 +01001530 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1531
Eric Anholtce44b0e2008-11-06 16:00:31 -08001532 obj_priv->last_rendering_seqno = 0;
Zou Nan hai852835f2010-05-21 09:08:56 +08001533 obj_priv->ring = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07001534 if (obj_priv->active) {
1535 obj_priv->active = 0;
1536 drm_gem_object_unreference(obj);
1537 }
1538 i915_verify_inactive(dev, __FILE__, __LINE__);
1539}
1540
Daniel Vetter63560392010-02-19 11:51:59 +01001541static void
1542i915_gem_process_flushing_list(struct drm_device *dev,
Zou Nan hai852835f2010-05-21 09:08:56 +08001543 uint32_t flush_domains, uint32_t seqno,
1544 struct intel_ring_buffer *ring)
Daniel Vetter63560392010-02-19 11:51:59 +01001545{
1546 drm_i915_private_t *dev_priv = dev->dev_private;
1547 struct drm_i915_gem_object *obj_priv, *next;
1548
1549 list_for_each_entry_safe(obj_priv, next,
1550 &dev_priv->mm.gpu_write_list,
1551 gpu_write_list) {
Daniel Vettera8089e82010-04-09 19:05:09 +00001552 struct drm_gem_object *obj = &obj_priv->base;
Daniel Vetter63560392010-02-19 11:51:59 +01001553
1554 if ((obj->write_domain & flush_domains) ==
Zou Nan hai852835f2010-05-21 09:08:56 +08001555 obj->write_domain &&
1556 obj_priv->ring->ring_flag == ring->ring_flag) {
Daniel Vetter63560392010-02-19 11:51:59 +01001557 uint32_t old_write_domain = obj->write_domain;
1558
1559 obj->write_domain = 0;
1560 list_del_init(&obj_priv->gpu_write_list);
Zou Nan hai852835f2010-05-21 09:08:56 +08001561 i915_gem_object_move_to_active(obj, seqno, ring);
Daniel Vetter63560392010-02-19 11:51:59 +01001562
1563 /* update the fence lru list */
Daniel Vetter007cc8a2010-04-28 11:02:31 +02001564 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1565 struct drm_i915_fence_reg *reg =
1566 &dev_priv->fence_regs[obj_priv->fence_reg];
1567 list_move_tail(&reg->lru_list,
Daniel Vetter63560392010-02-19 11:51:59 +01001568 &dev_priv->mm.fence_list);
Daniel Vetter007cc8a2010-04-28 11:02:31 +02001569 }
Daniel Vetter63560392010-02-19 11:51:59 +01001570
1571 trace_i915_gem_object_change_domain(obj,
1572 obj->read_domains,
1573 old_write_domain);
1574 }
1575 }
1576}
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001577
Daniel Vetter5a5a0c62009-09-15 22:57:36 +02001578uint32_t
Eric Anholtb9624422009-06-03 07:27:35 +00001579i915_add_request(struct drm_device *dev, struct drm_file *file_priv,
Zou Nan hai852835f2010-05-21 09:08:56 +08001580 uint32_t flush_domains, struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001581{
1582 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholtb9624422009-06-03 07:27:35 +00001583 struct drm_i915_file_private *i915_file_priv = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07001584 struct drm_i915_gem_request *request;
1585 uint32_t seqno;
1586 int was_empty;
Eric Anholt673a3942008-07-30 12:06:12 -07001587
Eric Anholtb9624422009-06-03 07:27:35 +00001588 if (file_priv != NULL)
1589 i915_file_priv = file_priv->driver_priv;
1590
Eric Anholt9a298b22009-03-24 12:23:04 -07001591 request = kzalloc(sizeof(*request), GFP_KERNEL);
Eric Anholt673a3942008-07-30 12:06:12 -07001592 if (request == NULL)
1593 return 0;
1594
Zou Nan hai852835f2010-05-21 09:08:56 +08001595 seqno = ring->add_request(dev, ring, file_priv, flush_domains);
Eric Anholt673a3942008-07-30 12:06:12 -07001596
1597 request->seqno = seqno;
Zou Nan hai852835f2010-05-21 09:08:56 +08001598 request->ring = ring;
Eric Anholt673a3942008-07-30 12:06:12 -07001599 request->emitted_jiffies = jiffies;
Zou Nan hai852835f2010-05-21 09:08:56 +08001600 was_empty = list_empty(&ring->request_list);
1601 list_add_tail(&request->list, &ring->request_list);
1602
Eric Anholtb9624422009-06-03 07:27:35 +00001603 if (i915_file_priv) {
1604 list_add_tail(&request->client_list,
1605 &i915_file_priv->mm.request_list);
1606 } else {
1607 INIT_LIST_HEAD(&request->client_list);
1608 }
Eric Anholt673a3942008-07-30 12:06:12 -07001609
Eric Anholtce44b0e2008-11-06 16:00:31 -08001610 /* Associate any objects on the flushing list matching the write
1611 * domain we're flushing with our flush.
1612 */
Daniel Vetter63560392010-02-19 11:51:59 +01001613 if (flush_domains != 0)
Zou Nan hai852835f2010-05-21 09:08:56 +08001614 i915_gem_process_flushing_list(dev, flush_domains, seqno, ring);
Eric Anholtce44b0e2008-11-06 16:00:31 -08001615
Ben Gamarif65d9422009-09-14 17:48:44 -04001616 if (!dev_priv->mm.suspended) {
1617 mod_timer(&dev_priv->hangcheck_timer, jiffies + DRM_I915_HANGCHECK_PERIOD);
1618 if (was_empty)
1619 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1620 }
Eric Anholt673a3942008-07-30 12:06:12 -07001621 return seqno;
1622}
1623
1624/**
1625 * Command execution barrier
1626 *
1627 * Ensures that all commands in the ring are finished
1628 * before signalling the CPU
1629 */
Eric Anholt3043c602008-10-02 12:24:47 -07001630static uint32_t
Zou Nan hai852835f2010-05-21 09:08:56 +08001631i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001632{
Eric Anholt673a3942008-07-30 12:06:12 -07001633 uint32_t flush_domains = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07001634
1635 /* The sampler always gets flushed on i965 (sigh) */
1636 if (IS_I965G(dev))
1637 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
Zou Nan hai852835f2010-05-21 09:08:56 +08001638
1639 ring->flush(dev, ring,
1640 I915_GEM_DOMAIN_COMMAND, flush_domains);
Eric Anholt673a3942008-07-30 12:06:12 -07001641 return flush_domains;
1642}
1643
1644/**
1645 * Moves buffers associated only with the given active seqno from the active
1646 * to inactive list, potentially freeing them.
1647 */
1648static void
1649i915_gem_retire_request(struct drm_device *dev,
1650 struct drm_i915_gem_request *request)
1651{
1652 drm_i915_private_t *dev_priv = dev->dev_private;
1653
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001654 trace_i915_gem_request_retire(dev, request->seqno);
1655
Eric Anholt673a3942008-07-30 12:06:12 -07001656 /* Move any buffers on the active list that are no longer referenced
1657 * by the ringbuffer to the flushing/inactive lists as appropriate.
1658 */
Carl Worth5e118f42009-03-20 11:54:25 -07001659 spin_lock(&dev_priv->mm.active_list_lock);
Zou Nan hai852835f2010-05-21 09:08:56 +08001660 while (!list_empty(&request->ring->active_list)) {
Eric Anholt673a3942008-07-30 12:06:12 -07001661 struct drm_gem_object *obj;
1662 struct drm_i915_gem_object *obj_priv;
1663
Zou Nan hai852835f2010-05-21 09:08:56 +08001664 obj_priv = list_first_entry(&request->ring->active_list,
Eric Anholt673a3942008-07-30 12:06:12 -07001665 struct drm_i915_gem_object,
1666 list);
Daniel Vettera8089e82010-04-09 19:05:09 +00001667 obj = &obj_priv->base;
Eric Anholt673a3942008-07-30 12:06:12 -07001668
1669 /* If the seqno being retired doesn't match the oldest in the
1670 * list, then the oldest in the list must still be newer than
1671 * this seqno.
1672 */
1673 if (obj_priv->last_rendering_seqno != request->seqno)
Carl Worth5e118f42009-03-20 11:54:25 -07001674 goto out;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001675
Eric Anholt673a3942008-07-30 12:06:12 -07001676#if WATCH_LRU
1677 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1678 __func__, request->seqno, obj);
1679#endif
1680
Eric Anholtce44b0e2008-11-06 16:00:31 -08001681 if (obj->write_domain != 0)
1682 i915_gem_object_move_to_flushing(obj);
Shaohua Li68c84342009-04-08 10:58:23 +08001683 else {
1684 /* Take a reference on the object so it won't be
1685 * freed while the spinlock is held. The list
1686 * protection for this spinlock is safe when breaking
1687 * the lock like this since the next thing we do
1688 * is just get the head of the list again.
1689 */
1690 drm_gem_object_reference(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001691 i915_gem_object_move_to_inactive(obj);
Shaohua Li68c84342009-04-08 10:58:23 +08001692 spin_unlock(&dev_priv->mm.active_list_lock);
1693 drm_gem_object_unreference(obj);
1694 spin_lock(&dev_priv->mm.active_list_lock);
1695 }
Eric Anholt673a3942008-07-30 12:06:12 -07001696 }
Carl Worth5e118f42009-03-20 11:54:25 -07001697out:
1698 spin_unlock(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07001699}
1700
1701/**
1702 * Returns true if seq1 is later than seq2.
1703 */
Ben Gamari22be1722009-09-14 17:48:43 -04001704bool
Eric Anholt673a3942008-07-30 12:06:12 -07001705i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1706{
1707 return (int32_t)(seq1 - seq2) >= 0;
1708}
1709
1710uint32_t
Zou Nan hai852835f2010-05-21 09:08:56 +08001711i915_get_gem_seqno(struct drm_device *dev,
Zou Nan haid1b851f2010-05-21 09:08:57 +08001712 struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001713{
Zou Nan hai852835f2010-05-21 09:08:56 +08001714 return ring->get_gem_seqno(dev, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07001715}
1716
1717/**
1718 * This function clears the request list as sequence numbers are passed.
1719 */
1720void
Zou Nan hai852835f2010-05-21 09:08:56 +08001721i915_gem_retire_requests(struct drm_device *dev,
1722 struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001723{
1724 drm_i915_private_t *dev_priv = dev->dev_private;
1725 uint32_t seqno;
1726
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001727 if (!ring->status_page.page_addr
Zou Nan hai852835f2010-05-21 09:08:56 +08001728 || list_empty(&ring->request_list))
Karsten Wiese6c0594a2009-02-23 15:07:57 +01001729 return;
1730
Zou Nan hai852835f2010-05-21 09:08:56 +08001731 seqno = i915_get_gem_seqno(dev, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07001732
Zou Nan hai852835f2010-05-21 09:08:56 +08001733 while (!list_empty(&ring->request_list)) {
Eric Anholt673a3942008-07-30 12:06:12 -07001734 struct drm_i915_gem_request *request;
1735 uint32_t retiring_seqno;
1736
Zou Nan hai852835f2010-05-21 09:08:56 +08001737 request = list_first_entry(&ring->request_list,
Eric Anholt673a3942008-07-30 12:06:12 -07001738 struct drm_i915_gem_request,
1739 list);
1740 retiring_seqno = request->seqno;
1741
1742 if (i915_seqno_passed(seqno, retiring_seqno) ||
Ben Gamariba1234d2009-09-14 17:48:47 -04001743 atomic_read(&dev_priv->mm.wedged)) {
Eric Anholt673a3942008-07-30 12:06:12 -07001744 i915_gem_retire_request(dev, request);
1745
1746 list_del(&request->list);
Eric Anholtb9624422009-06-03 07:27:35 +00001747 list_del(&request->client_list);
Eric Anholt9a298b22009-03-24 12:23:04 -07001748 kfree(request);
Eric Anholt673a3942008-07-30 12:06:12 -07001749 } else
1750 break;
1751 }
Chris Wilson9d34e5d2009-09-24 05:26:06 +01001752
1753 if (unlikely (dev_priv->trace_irq_seqno &&
1754 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001755
1756 ring->user_irq_put(dev, ring);
Chris Wilson9d34e5d2009-09-24 05:26:06 +01001757 dev_priv->trace_irq_seqno = 0;
1758 }
Eric Anholt673a3942008-07-30 12:06:12 -07001759}
1760
1761void
1762i915_gem_retire_work_handler(struct work_struct *work)
1763{
1764 drm_i915_private_t *dev_priv;
1765 struct drm_device *dev;
1766
1767 dev_priv = container_of(work, drm_i915_private_t,
1768 mm.retire_work.work);
1769 dev = dev_priv->dev;
1770
1771 mutex_lock(&dev->struct_mutex);
Zou Nan hai852835f2010-05-21 09:08:56 +08001772 i915_gem_retire_requests(dev, &dev_priv->render_ring);
1773
Zou Nan haid1b851f2010-05-21 09:08:57 +08001774 if (HAS_BSD(dev))
1775 i915_gem_retire_requests(dev, &dev_priv->bsd_ring);
1776
Keith Packard6dbe2772008-10-14 21:41:13 -07001777 if (!dev_priv->mm.suspended &&
Zou Nan haid1b851f2010-05-21 09:08:57 +08001778 (!list_empty(&dev_priv->render_ring.request_list) ||
1779 (HAS_BSD(dev) &&
1780 !list_empty(&dev_priv->bsd_ring.request_list))))
Eric Anholt9c9fe1f2009-08-03 16:09:16 -07001781 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
Eric Anholt673a3942008-07-30 12:06:12 -07001782 mutex_unlock(&dev->struct_mutex);
1783}
1784
Daniel Vetter5a5a0c62009-09-15 22:57:36 +02001785int
Zou Nan hai852835f2010-05-21 09:08:56 +08001786i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
1787 int interruptible, struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001788{
1789 drm_i915_private_t *dev_priv = dev->dev_private;
Jesse Barnes802c7eb2009-05-05 16:03:48 -07001790 u32 ier;
Eric Anholt673a3942008-07-30 12:06:12 -07001791 int ret = 0;
1792
1793 BUG_ON(seqno == 0);
1794
Ben Gamariba1234d2009-09-14 17:48:47 -04001795 if (atomic_read(&dev_priv->mm.wedged))
Ben Gamariffed1d02009-09-14 17:48:41 -04001796 return -EIO;
1797
Zou Nan hai852835f2010-05-21 09:08:56 +08001798 if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
Eric Anholtbad720f2009-10-22 16:11:14 -07001799 if (HAS_PCH_SPLIT(dev))
Zhenyu Wang036a4a72009-06-08 14:40:19 +08001800 ier = I915_READ(DEIER) | I915_READ(GTIER);
1801 else
1802 ier = I915_READ(IER);
Jesse Barnes802c7eb2009-05-05 16:03:48 -07001803 if (!ier) {
1804 DRM_ERROR("something (likely vbetool) disabled "
1805 "interrupts, re-enabling\n");
1806 i915_driver_irq_preinstall(dev);
1807 i915_driver_irq_postinstall(dev);
1808 }
1809
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001810 trace_i915_gem_request_wait_begin(dev, seqno);
1811
Zou Nan hai852835f2010-05-21 09:08:56 +08001812 ring->waiting_gem_seqno = seqno;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001813 ring->user_irq_get(dev, ring);
Daniel Vetter48764bf2009-09-15 22:57:32 +02001814 if (interruptible)
Zou Nan hai852835f2010-05-21 09:08:56 +08001815 ret = wait_event_interruptible(ring->irq_queue,
1816 i915_seqno_passed(
1817 ring->get_gem_seqno(dev, ring), seqno)
1818 || atomic_read(&dev_priv->mm.wedged));
Daniel Vetter48764bf2009-09-15 22:57:32 +02001819 else
Zou Nan hai852835f2010-05-21 09:08:56 +08001820 wait_event(ring->irq_queue,
1821 i915_seqno_passed(
1822 ring->get_gem_seqno(dev, ring), seqno)
1823 || atomic_read(&dev_priv->mm.wedged));
Daniel Vetter48764bf2009-09-15 22:57:32 +02001824
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001825 ring->user_irq_put(dev, ring);
Zou Nan hai852835f2010-05-21 09:08:56 +08001826 ring->waiting_gem_seqno = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001827
1828 trace_i915_gem_request_wait_end(dev, seqno);
Eric Anholt673a3942008-07-30 12:06:12 -07001829 }
Ben Gamariba1234d2009-09-14 17:48:47 -04001830 if (atomic_read(&dev_priv->mm.wedged))
Eric Anholt673a3942008-07-30 12:06:12 -07001831 ret = -EIO;
1832
1833 if (ret && ret != -ERESTARTSYS)
1834 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
Zou Nan hai852835f2010-05-21 09:08:56 +08001835 __func__, ret, seqno, ring->get_gem_seqno(dev, ring));
Eric Anholt673a3942008-07-30 12:06:12 -07001836
1837 /* Directly dispatch request retiring. While we have the work queue
1838 * to handle this, the waiter on a request often wants an associated
1839 * buffer to have made it to the inactive list, and we would need
1840 * a separate wait queue to handle that.
1841 */
1842 if (ret == 0)
Zou Nan hai852835f2010-05-21 09:08:56 +08001843 i915_gem_retire_requests(dev, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07001844
1845 return ret;
1846}
1847
Daniel Vetter48764bf2009-09-15 22:57:32 +02001848/**
1849 * Waits for a sequence number to be signaled, and cleans up the
1850 * request and object lists appropriately for that event.
1851 */
1852static int
Zou Nan hai852835f2010-05-21 09:08:56 +08001853i915_wait_request(struct drm_device *dev, uint32_t seqno,
1854 struct intel_ring_buffer *ring)
Daniel Vetter48764bf2009-09-15 22:57:32 +02001855{
Zou Nan hai852835f2010-05-21 09:08:56 +08001856 return i915_do_wait_request(dev, seqno, 1, ring);
Daniel Vetter48764bf2009-09-15 22:57:32 +02001857}
1858
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001859static void
1860i915_gem_flush(struct drm_device *dev,
1861 uint32_t invalidate_domains,
1862 uint32_t flush_domains)
1863{
1864 drm_i915_private_t *dev_priv = dev->dev_private;
1865 if (flush_domains & I915_GEM_DOMAIN_CPU)
1866 drm_agp_chipset_flush(dev);
1867 dev_priv->render_ring.flush(dev, &dev_priv->render_ring,
1868 invalidate_domains,
1869 flush_domains);
Zou Nan haid1b851f2010-05-21 09:08:57 +08001870
1871 if (HAS_BSD(dev))
1872 dev_priv->bsd_ring.flush(dev, &dev_priv->bsd_ring,
1873 invalidate_domains,
1874 flush_domains);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001875}
1876
Zou Nan hai852835f2010-05-21 09:08:56 +08001877static void
1878i915_gem_flush_ring(struct drm_device *dev,
1879 uint32_t invalidate_domains,
1880 uint32_t flush_domains,
1881 struct intel_ring_buffer *ring)
1882{
1883 if (flush_domains & I915_GEM_DOMAIN_CPU)
1884 drm_agp_chipset_flush(dev);
1885 ring->flush(dev, ring,
1886 invalidate_domains,
1887 flush_domains);
1888}
1889
Eric Anholt673a3942008-07-30 12:06:12 -07001890/**
1891 * Ensures that all rendering to the object has completed and the object is
1892 * safe to unbind from the GTT or access from the CPU.
1893 */
1894static int
1895i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1896{
1897 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001898 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001899 int ret;
1900
Eric Anholte47c68e2008-11-14 13:35:19 -08001901 /* This function only exists to support waiting for existing rendering,
1902 * not for emitting required flushes.
Eric Anholt673a3942008-07-30 12:06:12 -07001903 */
Eric Anholte47c68e2008-11-14 13:35:19 -08001904 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
Eric Anholt673a3942008-07-30 12:06:12 -07001905
1906 /* If there is rendering queued on the buffer being evicted, wait for
1907 * it.
1908 */
1909 if (obj_priv->active) {
1910#if WATCH_BUF
1911 DRM_INFO("%s: object %p wait for seqno %08x\n",
1912 __func__, obj, obj_priv->last_rendering_seqno);
1913#endif
Zou Nan hai852835f2010-05-21 09:08:56 +08001914 ret = i915_wait_request(dev,
1915 obj_priv->last_rendering_seqno, obj_priv->ring);
Eric Anholt673a3942008-07-30 12:06:12 -07001916 if (ret != 0)
1917 return ret;
1918 }
1919
1920 return 0;
1921}
1922
1923/**
1924 * Unbinds an object from the GTT aperture.
1925 */
Jesse Barnes0f973f22009-01-26 17:10:45 -08001926int
Eric Anholt673a3942008-07-30 12:06:12 -07001927i915_gem_object_unbind(struct drm_gem_object *obj)
1928{
1929 struct drm_device *dev = obj->dev;
Daniel Vetter4a87b8c2010-02-19 11:51:57 +01001930 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001931 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001932 int ret = 0;
1933
1934#if WATCH_BUF
1935 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
1936 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
1937#endif
1938 if (obj_priv->gtt_space == NULL)
1939 return 0;
1940
1941 if (obj_priv->pin_count != 0) {
1942 DRM_ERROR("Attempting to unbind pinned buffer\n");
1943 return -EINVAL;
1944 }
1945
Eric Anholt5323fd02009-09-09 11:50:45 -07001946 /* blow away mappings if mapped through GTT */
1947 i915_gem_release_mmap(obj);
1948
Eric Anholt673a3942008-07-30 12:06:12 -07001949 /* Move the object to the CPU domain to ensure that
1950 * any possible CPU writes while it's not in the GTT
1951 * are flushed when we go to remap it. This will
1952 * also ensure that all pending GPU writes are finished
1953 * before we unbind.
1954 */
Eric Anholte47c68e2008-11-14 13:35:19 -08001955 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
Eric Anholt673a3942008-07-30 12:06:12 -07001956 if (ret) {
Eric Anholte47c68e2008-11-14 13:35:19 -08001957 if (ret != -ERESTARTSYS)
1958 DRM_ERROR("set_domain failed: %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07001959 return ret;
1960 }
1961
Eric Anholt5323fd02009-09-09 11:50:45 -07001962 BUG_ON(obj_priv->active);
1963
Daniel Vetter96b47b62009-12-15 17:50:00 +01001964 /* release the fence reg _after_ flushing */
1965 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
1966 i915_gem_clear_fence_reg(obj);
1967
Eric Anholt673a3942008-07-30 12:06:12 -07001968 if (obj_priv->agp_mem != NULL) {
1969 drm_unbind_agp(obj_priv->agp_mem);
1970 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
1971 obj_priv->agp_mem = NULL;
1972 }
1973
Eric Anholt856fa192009-03-19 14:10:50 -07001974 i915_gem_object_put_pages(obj);
Chris Wilsona32808c2009-09-20 21:29:47 +01001975 BUG_ON(obj_priv->pages_refcount);
Eric Anholt673a3942008-07-30 12:06:12 -07001976
1977 if (obj_priv->gtt_space) {
1978 atomic_dec(&dev->gtt_count);
1979 atomic_sub(obj->size, &dev->gtt_memory);
1980
1981 drm_mm_put_block(obj_priv->gtt_space);
1982 obj_priv->gtt_space = NULL;
1983 }
1984
1985 /* Remove ourselves from the LRU list if present. */
Daniel Vetter4a87b8c2010-02-19 11:51:57 +01001986 spin_lock(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07001987 if (!list_empty(&obj_priv->list))
1988 list_del_init(&obj_priv->list);
Daniel Vetter4a87b8c2010-02-19 11:51:57 +01001989 spin_unlock(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07001990
Chris Wilson963b4832009-09-20 23:03:54 +01001991 if (i915_gem_object_is_purgeable(obj_priv))
1992 i915_gem_object_truncate(obj);
1993
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001994 trace_i915_gem_object_unbind(obj);
1995
Eric Anholt673a3942008-07-30 12:06:12 -07001996 return 0;
1997}
1998
Chris Wilson07f73f62009-09-14 16:50:30 +01001999static struct drm_gem_object *
2000i915_gem_find_inactive_object(struct drm_device *dev, int min_size)
2001{
2002 drm_i915_private_t *dev_priv = dev->dev_private;
2003 struct drm_i915_gem_object *obj_priv;
2004 struct drm_gem_object *best = NULL;
2005 struct drm_gem_object *first = NULL;
2006
2007 /* Try to find the smallest clean object */
2008 list_for_each_entry(obj_priv, &dev_priv->mm.inactive_list, list) {
Daniel Vettera8089e82010-04-09 19:05:09 +00002009 struct drm_gem_object *obj = &obj_priv->base;
Chris Wilson07f73f62009-09-14 16:50:30 +01002010 if (obj->size >= min_size) {
Chris Wilson963b4832009-09-20 23:03:54 +01002011 if ((!obj_priv->dirty ||
2012 i915_gem_object_is_purgeable(obj_priv)) &&
Chris Wilson07f73f62009-09-14 16:50:30 +01002013 (!best || obj->size < best->size)) {
2014 best = obj;
2015 if (best->size == min_size)
2016 return best;
2017 }
2018 if (!first)
2019 first = obj;
2020 }
2021 }
2022
2023 return best ? best : first;
2024}
2025
Eric Anholt673a3942008-07-30 12:06:12 -07002026static int
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002027i915_gpu_idle(struct drm_device *dev)
2028{
2029 drm_i915_private_t *dev_priv = dev->dev_private;
2030 bool lists_empty;
Zou Nan haid1b851f2010-05-21 09:08:57 +08002031 uint32_t seqno1, seqno2;
Zou Nan hai852835f2010-05-21 09:08:56 +08002032 int ret;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002033
2034 spin_lock(&dev_priv->mm.active_list_lock);
Zou Nan haid1b851f2010-05-21 09:08:57 +08002035 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2036 list_empty(&dev_priv->render_ring.active_list) &&
2037 (!HAS_BSD(dev) ||
2038 list_empty(&dev_priv->bsd_ring.active_list)));
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002039 spin_unlock(&dev_priv->mm.active_list_lock);
2040
2041 if (lists_empty)
2042 return 0;
2043
2044 /* Flush everything onto the inactive list. */
2045 i915_gem_flush(dev, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
Zou Nan haid1b851f2010-05-21 09:08:57 +08002046 seqno1 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
Zou Nan hai852835f2010-05-21 09:08:56 +08002047 &dev_priv->render_ring);
Zou Nan haid1b851f2010-05-21 09:08:57 +08002048 if (seqno1 == 0)
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002049 return -ENOMEM;
Zou Nan haid1b851f2010-05-21 09:08:57 +08002050 ret = i915_wait_request(dev, seqno1, &dev_priv->render_ring);
2051
2052 if (HAS_BSD(dev)) {
2053 seqno2 = i915_add_request(dev, NULL, I915_GEM_GPU_DOMAINS,
2054 &dev_priv->bsd_ring);
2055 if (seqno2 == 0)
2056 return -ENOMEM;
2057
2058 ret = i915_wait_request(dev, seqno2, &dev_priv->bsd_ring);
2059 if (ret)
2060 return ret;
2061 }
2062
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002063
Zou Nan hai852835f2010-05-21 09:08:56 +08002064 return ret;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002065}
2066
2067static int
Chris Wilson07f73f62009-09-14 16:50:30 +01002068i915_gem_evict_everything(struct drm_device *dev)
2069{
2070 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson07f73f62009-09-14 16:50:30 +01002071 int ret;
2072 bool lists_empty;
2073
Chris Wilson07f73f62009-09-14 16:50:30 +01002074 spin_lock(&dev_priv->mm.active_list_lock);
2075 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2076 list_empty(&dev_priv->mm.flushing_list) &&
Zou Nan haid1b851f2010-05-21 09:08:57 +08002077 list_empty(&dev_priv->render_ring.active_list) &&
2078 (!HAS_BSD(dev)
2079 || list_empty(&dev_priv->bsd_ring.active_list)));
Chris Wilson07f73f62009-09-14 16:50:30 +01002080 spin_unlock(&dev_priv->mm.active_list_lock);
2081
Chris Wilson97311292009-09-21 00:22:34 +01002082 if (lists_empty)
Chris Wilson07f73f62009-09-14 16:50:30 +01002083 return -ENOSPC;
Chris Wilson07f73f62009-09-14 16:50:30 +01002084
2085 /* Flush everything (on to the inactive lists) and evict */
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002086 ret = i915_gpu_idle(dev);
Chris Wilson07f73f62009-09-14 16:50:30 +01002087 if (ret)
2088 return ret;
2089
Daniel Vetter99fcb762010-02-07 16:20:18 +01002090 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2091
Chris Wilsonab5ee572009-09-20 19:25:47 +01002092 ret = i915_gem_evict_from_inactive_list(dev);
Chris Wilson07f73f62009-09-14 16:50:30 +01002093 if (ret)
2094 return ret;
2095
2096 spin_lock(&dev_priv->mm.active_list_lock);
2097 lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
2098 list_empty(&dev_priv->mm.flushing_list) &&
Zou Nan haid1b851f2010-05-21 09:08:57 +08002099 list_empty(&dev_priv->render_ring.active_list) &&
2100 (!HAS_BSD(dev)
2101 || list_empty(&dev_priv->bsd_ring.active_list)));
Chris Wilson07f73f62009-09-14 16:50:30 +01002102 spin_unlock(&dev_priv->mm.active_list_lock);
2103 BUG_ON(!lists_empty);
2104
Eric Anholt673a3942008-07-30 12:06:12 -07002105 return 0;
2106}
2107
2108static int
Chris Wilson07f73f62009-09-14 16:50:30 +01002109i915_gem_evict_something(struct drm_device *dev, int min_size)
Eric Anholt673a3942008-07-30 12:06:12 -07002110{
2111 drm_i915_private_t *dev_priv = dev->dev_private;
2112 struct drm_gem_object *obj;
Chris Wilson07f73f62009-09-14 16:50:30 +01002113 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002114
Zou Nan hai852835f2010-05-21 09:08:56 +08002115 struct intel_ring_buffer *render_ring = &dev_priv->render_ring;
Zou Nan haid1b851f2010-05-21 09:08:57 +08002116 struct intel_ring_buffer *bsd_ring = &dev_priv->bsd_ring;
Eric Anholt673a3942008-07-30 12:06:12 -07002117 for (;;) {
Zou Nan hai852835f2010-05-21 09:08:56 +08002118 i915_gem_retire_requests(dev, render_ring);
Chris Wilson07f73f62009-09-14 16:50:30 +01002119
Zou Nan haid1b851f2010-05-21 09:08:57 +08002120 if (HAS_BSD(dev))
2121 i915_gem_retire_requests(dev, bsd_ring);
2122
Eric Anholt673a3942008-07-30 12:06:12 -07002123 /* If there's an inactive buffer available now, grab it
2124 * and be done.
2125 */
Chris Wilson07f73f62009-09-14 16:50:30 +01002126 obj = i915_gem_find_inactive_object(dev, min_size);
2127 if (obj) {
2128 struct drm_i915_gem_object *obj_priv;
2129
Eric Anholt673a3942008-07-30 12:06:12 -07002130#if WATCH_LRU
2131 DRM_INFO("%s: evicting %p\n", __func__, obj);
2132#endif
Daniel Vetter23010e42010-03-08 13:35:02 +01002133 obj_priv = to_intel_bo(obj);
Chris Wilson07f73f62009-09-14 16:50:30 +01002134 BUG_ON(obj_priv->pin_count != 0);
Eric Anholt673a3942008-07-30 12:06:12 -07002135 BUG_ON(obj_priv->active);
2136
2137 /* Wait on the rendering and unbind the buffer. */
Chris Wilson07f73f62009-09-14 16:50:30 +01002138 return i915_gem_object_unbind(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002139 }
2140
2141 /* If we didn't get anything, but the ring is still processing
Chris Wilson07f73f62009-09-14 16:50:30 +01002142 * things, wait for the next to finish and hopefully leave us
2143 * a buffer to evict.
Eric Anholt673a3942008-07-30 12:06:12 -07002144 */
Zou Nan hai852835f2010-05-21 09:08:56 +08002145 if (!list_empty(&render_ring->request_list)) {
Eric Anholt673a3942008-07-30 12:06:12 -07002146 struct drm_i915_gem_request *request;
2147
Zou Nan hai852835f2010-05-21 09:08:56 +08002148 request = list_first_entry(&render_ring->request_list,
Eric Anholt673a3942008-07-30 12:06:12 -07002149 struct drm_i915_gem_request,
2150 list);
2151
Zou Nan hai852835f2010-05-21 09:08:56 +08002152 ret = i915_wait_request(dev,
2153 request->seqno, request->ring);
Eric Anholt673a3942008-07-30 12:06:12 -07002154 if (ret)
Chris Wilson07f73f62009-09-14 16:50:30 +01002155 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002156
Chris Wilson07f73f62009-09-14 16:50:30 +01002157 continue;
Eric Anholt673a3942008-07-30 12:06:12 -07002158 }
2159
Zou Nan haid1b851f2010-05-21 09:08:57 +08002160 if (HAS_BSD(dev) && !list_empty(&bsd_ring->request_list)) {
2161 struct drm_i915_gem_request *request;
2162
2163 request = list_first_entry(&bsd_ring->request_list,
2164 struct drm_i915_gem_request,
2165 list);
2166
2167 ret = i915_wait_request(dev,
2168 request->seqno, request->ring);
2169 if (ret)
2170 return ret;
2171
2172 continue;
2173 }
2174
Eric Anholt673a3942008-07-30 12:06:12 -07002175 /* If we didn't have anything on the request list but there
2176 * are buffers awaiting a flush, emit one and try again.
2177 * When we wait on it, those buffers waiting for that flush
2178 * will get moved to inactive.
2179 */
2180 if (!list_empty(&dev_priv->mm.flushing_list)) {
Chris Wilson07f73f62009-09-14 16:50:30 +01002181 struct drm_i915_gem_object *obj_priv;
Eric Anholt673a3942008-07-30 12:06:12 -07002182
Chris Wilson9a1e2582009-09-20 20:16:50 +01002183 /* Find an object that we can immediately reuse */
2184 list_for_each_entry(obj_priv, &dev_priv->mm.flushing_list, list) {
Daniel Vettera8089e82010-04-09 19:05:09 +00002185 obj = &obj_priv->base;
Chris Wilson9a1e2582009-09-20 20:16:50 +01002186 if (obj->size >= min_size)
2187 break;
Eric Anholt673a3942008-07-30 12:06:12 -07002188
Chris Wilson9a1e2582009-09-20 20:16:50 +01002189 obj = NULL;
2190 }
Eric Anholt673a3942008-07-30 12:06:12 -07002191
Chris Wilson9a1e2582009-09-20 20:16:50 +01002192 if (obj != NULL) {
2193 uint32_t seqno;
Chris Wilson07f73f62009-09-14 16:50:30 +01002194
Zou Nan hai852835f2010-05-21 09:08:56 +08002195 i915_gem_flush_ring(dev,
Chris Wilson9a1e2582009-09-20 20:16:50 +01002196 obj->write_domain,
Zou Nan hai852835f2010-05-21 09:08:56 +08002197 obj->write_domain,
2198 obj_priv->ring);
2199 seqno = i915_add_request(dev, NULL,
2200 obj->write_domain,
2201 obj_priv->ring);
Chris Wilson9a1e2582009-09-20 20:16:50 +01002202 if (seqno == 0)
2203 return -ENOMEM;
Chris Wilson9a1e2582009-09-20 20:16:50 +01002204 continue;
2205 }
Eric Anholt673a3942008-07-30 12:06:12 -07002206 }
2207
Chris Wilson07f73f62009-09-14 16:50:30 +01002208 /* If we didn't do any of the above, there's no single buffer
2209 * large enough to swap out for the new one, so just evict
2210 * everything and start again. (This should be rare.)
Eric Anholt673a3942008-07-30 12:06:12 -07002211 */
Chris Wilson97311292009-09-21 00:22:34 +01002212 if (!list_empty (&dev_priv->mm.inactive_list))
Chris Wilsonab5ee572009-09-20 19:25:47 +01002213 return i915_gem_evict_from_inactive_list(dev);
Chris Wilson97311292009-09-21 00:22:34 +01002214 else
Chris Wilson07f73f62009-09-14 16:50:30 +01002215 return i915_gem_evict_everything(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07002216 }
Keith Packardac94a962008-11-20 23:30:27 -08002217}
2218
Ben Gamari6911a9b2009-04-02 11:24:54 -07002219int
Chris Wilson4bdadb92010-01-27 13:36:32 +00002220i915_gem_object_get_pages(struct drm_gem_object *obj,
2221 gfp_t gfpmask)
Eric Anholt673a3942008-07-30 12:06:12 -07002222{
Daniel Vetter23010e42010-03-08 13:35:02 +01002223 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002224 int page_count, i;
2225 struct address_space *mapping;
2226 struct inode *inode;
2227 struct page *page;
Eric Anholt673a3942008-07-30 12:06:12 -07002228
Daniel Vetter778c3542010-05-13 11:49:44 +02002229 BUG_ON(obj_priv->pages_refcount
2230 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2231
Eric Anholt856fa192009-03-19 14:10:50 -07002232 if (obj_priv->pages_refcount++ != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07002233 return 0;
2234
2235 /* Get the list of pages out of our struct file. They'll be pinned
2236 * at this point until we release them.
2237 */
2238 page_count = obj->size / PAGE_SIZE;
Eric Anholt856fa192009-03-19 14:10:50 -07002239 BUG_ON(obj_priv->pages != NULL);
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07002240 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
Eric Anholt856fa192009-03-19 14:10:50 -07002241 if (obj_priv->pages == NULL) {
Eric Anholt856fa192009-03-19 14:10:50 -07002242 obj_priv->pages_refcount--;
Eric Anholt673a3942008-07-30 12:06:12 -07002243 return -ENOMEM;
2244 }
2245
2246 inode = obj->filp->f_path.dentry->d_inode;
2247 mapping = inode->i_mapping;
2248 for (i = 0; i < page_count; i++) {
Chris Wilson4bdadb92010-01-27 13:36:32 +00002249 page = read_cache_page_gfp(mapping, i,
2250 mapping_gfp_mask (mapping) |
2251 __GFP_COLD |
2252 gfpmask);
Chris Wilson1f2b1012010-03-12 19:52:55 +00002253 if (IS_ERR(page))
2254 goto err_pages;
2255
Eric Anholt856fa192009-03-19 14:10:50 -07002256 obj_priv->pages[i] = page;
Eric Anholt673a3942008-07-30 12:06:12 -07002257 }
Eric Anholt280b7132009-03-12 16:56:27 -07002258
2259 if (obj_priv->tiling_mode != I915_TILING_NONE)
2260 i915_gem_object_do_bit_17_swizzle(obj);
2261
Eric Anholt673a3942008-07-30 12:06:12 -07002262 return 0;
Chris Wilson1f2b1012010-03-12 19:52:55 +00002263
2264err_pages:
2265 while (i--)
2266 page_cache_release(obj_priv->pages[i]);
2267
2268 drm_free_large(obj_priv->pages);
2269 obj_priv->pages = NULL;
2270 obj_priv->pages_refcount--;
2271 return PTR_ERR(page);
Eric Anholt673a3942008-07-30 12:06:12 -07002272}
2273
Eric Anholt4e901fd2009-10-26 16:44:17 -07002274static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2275{
2276 struct drm_gem_object *obj = reg->obj;
2277 struct drm_device *dev = obj->dev;
2278 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002279 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt4e901fd2009-10-26 16:44:17 -07002280 int regnum = obj_priv->fence_reg;
2281 uint64_t val;
2282
2283 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2284 0xfffff000) << 32;
2285 val |= obj_priv->gtt_offset & 0xfffff000;
2286 val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2287 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2288
2289 if (obj_priv->tiling_mode == I915_TILING_Y)
2290 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2291 val |= I965_FENCE_REG_VALID;
2292
2293 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2294}
2295
Jesse Barnesde151cf2008-11-12 10:03:55 -08002296static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2297{
2298 struct drm_gem_object *obj = reg->obj;
2299 struct drm_device *dev = obj->dev;
2300 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002301 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002302 int regnum = obj_priv->fence_reg;
2303 uint64_t val;
2304
2305 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2306 0xfffff000) << 32;
2307 val |= obj_priv->gtt_offset & 0xfffff000;
2308 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2309 if (obj_priv->tiling_mode == I915_TILING_Y)
2310 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2311 val |= I965_FENCE_REG_VALID;
2312
2313 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2314}
2315
2316static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2317{
2318 struct drm_gem_object *obj = reg->obj;
2319 struct drm_device *dev = obj->dev;
2320 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002321 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002322 int regnum = obj_priv->fence_reg;
Jesse Barnes0f973f22009-01-26 17:10:45 -08002323 int tile_width;
Eric Anholtdc529a42009-03-10 22:34:49 -07002324 uint32_t fence_reg, val;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002325 uint32_t pitch_val;
2326
2327 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2328 (obj_priv->gtt_offset & (obj->size - 1))) {
Linus Torvaldsf06da262009-02-09 08:57:29 -08002329 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
Jesse Barnes0f973f22009-01-26 17:10:45 -08002330 __func__, obj_priv->gtt_offset, obj->size);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002331 return;
2332 }
2333
Jesse Barnes0f973f22009-01-26 17:10:45 -08002334 if (obj_priv->tiling_mode == I915_TILING_Y &&
2335 HAS_128_BYTE_Y_TILING(dev))
2336 tile_width = 128;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002337 else
Jesse Barnes0f973f22009-01-26 17:10:45 -08002338 tile_width = 512;
2339
2340 /* Note: pitch better be a power of two tile widths */
2341 pitch_val = obj_priv->stride / tile_width;
2342 pitch_val = ffs(pitch_val) - 1;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002343
Daniel Vetterc36a2a62010-04-17 15:12:03 +02002344 if (obj_priv->tiling_mode == I915_TILING_Y &&
2345 HAS_128_BYTE_Y_TILING(dev))
2346 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2347 else
2348 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2349
Jesse Barnesde151cf2008-11-12 10:03:55 -08002350 val = obj_priv->gtt_offset;
2351 if (obj_priv->tiling_mode == I915_TILING_Y)
2352 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2353 val |= I915_FENCE_SIZE_BITS(obj->size);
2354 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2355 val |= I830_FENCE_REG_VALID;
2356
Eric Anholtdc529a42009-03-10 22:34:49 -07002357 if (regnum < 8)
2358 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2359 else
2360 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2361 I915_WRITE(fence_reg, val);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002362}
2363
2364static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2365{
2366 struct drm_gem_object *obj = reg->obj;
2367 struct drm_device *dev = obj->dev;
2368 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002369 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002370 int regnum = obj_priv->fence_reg;
2371 uint32_t val;
2372 uint32_t pitch_val;
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002373 uint32_t fence_size_bits;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002374
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002375 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
Jesse Barnesde151cf2008-11-12 10:03:55 -08002376 (obj_priv->gtt_offset & (obj->size - 1))) {
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002377 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
Jesse Barnes0f973f22009-01-26 17:10:45 -08002378 __func__, obj_priv->gtt_offset);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002379 return;
2380 }
2381
Eric Anholte76a16d2009-05-26 17:44:56 -07002382 pitch_val = obj_priv->stride / 128;
2383 pitch_val = ffs(pitch_val) - 1;
2384 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2385
Jesse Barnesde151cf2008-11-12 10:03:55 -08002386 val = obj_priv->gtt_offset;
2387 if (obj_priv->tiling_mode == I915_TILING_Y)
2388 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002389 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2390 WARN_ON(fence_size_bits & ~0x00000f00);
2391 val |= fence_size_bits;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002392 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2393 val |= I830_FENCE_REG_VALID;
2394
2395 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002396}
2397
Daniel Vetterae3db242010-02-19 11:51:58 +01002398static int i915_find_fence_reg(struct drm_device *dev)
2399{
2400 struct drm_i915_fence_reg *reg = NULL;
2401 struct drm_i915_gem_object *obj_priv = NULL;
2402 struct drm_i915_private *dev_priv = dev->dev_private;
2403 struct drm_gem_object *obj = NULL;
2404 int i, avail, ret;
2405
2406 /* First try to find a free reg */
2407 avail = 0;
2408 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2409 reg = &dev_priv->fence_regs[i];
2410 if (!reg->obj)
2411 return i;
2412
Daniel Vetter23010e42010-03-08 13:35:02 +01002413 obj_priv = to_intel_bo(reg->obj);
Daniel Vetterae3db242010-02-19 11:51:58 +01002414 if (!obj_priv->pin_count)
2415 avail++;
2416 }
2417
2418 if (avail == 0)
2419 return -ENOSPC;
2420
2421 /* None available, try to steal one or wait for a user to finish */
2422 i = I915_FENCE_REG_NONE;
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002423 list_for_each_entry(reg, &dev_priv->mm.fence_list,
2424 lru_list) {
2425 obj = reg->obj;
2426 obj_priv = to_intel_bo(obj);
Daniel Vetterae3db242010-02-19 11:51:58 +01002427
2428 if (obj_priv->pin_count)
2429 continue;
2430
2431 /* found one! */
2432 i = obj_priv->fence_reg;
2433 break;
2434 }
2435
2436 BUG_ON(i == I915_FENCE_REG_NONE);
2437
2438 /* We only have a reference on obj from the active list. put_fence_reg
2439 * might drop that one, causing a use-after-free in it. So hold a
2440 * private reference to obj like the other callers of put_fence_reg
2441 * (set_tiling ioctl) do. */
2442 drm_gem_object_reference(obj);
2443 ret = i915_gem_object_put_fence_reg(obj);
2444 drm_gem_object_unreference(obj);
2445 if (ret != 0)
2446 return ret;
2447
2448 return i;
2449}
2450
Jesse Barnesde151cf2008-11-12 10:03:55 -08002451/**
2452 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2453 * @obj: object to map through a fence reg
2454 *
2455 * When mapping objects through the GTT, userspace wants to be able to write
2456 * to them without having to worry about swizzling if the object is tiled.
2457 *
2458 * This function walks the fence regs looking for a free one for @obj,
2459 * stealing one if it can't find any.
2460 *
2461 * It then sets up the reg based on the object's properties: address, pitch
2462 * and tiling format.
2463 */
Chris Wilson8c4b8c32009-06-17 22:08:52 +01002464int
2465i915_gem_object_get_fence_reg(struct drm_gem_object *obj)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002466{
2467 struct drm_device *dev = obj->dev;
Jesse Barnes79e53942008-11-07 14:24:08 -08002468 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002469 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002470 struct drm_i915_fence_reg *reg = NULL;
Daniel Vetterae3db242010-02-19 11:51:58 +01002471 int ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002472
Eric Anholta09ba7f2009-08-29 12:49:51 -07002473 /* Just update our place in the LRU if our fence is getting used. */
2474 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002475 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2476 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07002477 return 0;
2478 }
2479
Jesse Barnesde151cf2008-11-12 10:03:55 -08002480 switch (obj_priv->tiling_mode) {
2481 case I915_TILING_NONE:
2482 WARN(1, "allocating a fence for non-tiled object?\n");
2483 break;
2484 case I915_TILING_X:
Jesse Barnes0f973f22009-01-26 17:10:45 -08002485 if (!obj_priv->stride)
2486 return -EINVAL;
2487 WARN((obj_priv->stride & (512 - 1)),
2488 "object 0x%08x is X tiled but has non-512B pitch\n",
2489 obj_priv->gtt_offset);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002490 break;
2491 case I915_TILING_Y:
Jesse Barnes0f973f22009-01-26 17:10:45 -08002492 if (!obj_priv->stride)
2493 return -EINVAL;
2494 WARN((obj_priv->stride & (128 - 1)),
2495 "object 0x%08x is Y tiled but has non-128B pitch\n",
2496 obj_priv->gtt_offset);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002497 break;
2498 }
2499
Daniel Vetterae3db242010-02-19 11:51:58 +01002500 ret = i915_find_fence_reg(dev);
2501 if (ret < 0)
2502 return ret;
Chris Wilsonfc7170b2009-02-11 14:26:46 +00002503
Daniel Vetterae3db242010-02-19 11:51:58 +01002504 obj_priv->fence_reg = ret;
2505 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002506 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07002507
Jesse Barnesde151cf2008-11-12 10:03:55 -08002508 reg->obj = obj;
2509
Eric Anholt4e901fd2009-10-26 16:44:17 -07002510 if (IS_GEN6(dev))
2511 sandybridge_write_fence_reg(reg);
2512 else if (IS_I965G(dev))
Jesse Barnesde151cf2008-11-12 10:03:55 -08002513 i965_write_fence_reg(reg);
2514 else if (IS_I9XX(dev))
2515 i915_write_fence_reg(reg);
2516 else
2517 i830_write_fence_reg(reg);
Eric Anholtd9ddcb92009-01-27 10:33:49 -08002518
Daniel Vetterae3db242010-02-19 11:51:58 +01002519 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2520 obj_priv->tiling_mode);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002521
Eric Anholtd9ddcb92009-01-27 10:33:49 -08002522 return 0;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002523}
2524
2525/**
2526 * i915_gem_clear_fence_reg - clear out fence register info
2527 * @obj: object to clear
2528 *
2529 * Zeroes out the fence register itself and clears out the associated
2530 * data structures in dev_priv and obj_priv.
2531 */
2532static void
2533i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2534{
2535 struct drm_device *dev = obj->dev;
Jesse Barnes79e53942008-11-07 14:24:08 -08002536 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002537 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002538 struct drm_i915_fence_reg *reg =
2539 &dev_priv->fence_regs[obj_priv->fence_reg];
Jesse Barnesde151cf2008-11-12 10:03:55 -08002540
Eric Anholt4e901fd2009-10-26 16:44:17 -07002541 if (IS_GEN6(dev)) {
2542 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2543 (obj_priv->fence_reg * 8), 0);
2544 } else if (IS_I965G(dev)) {
Jesse Barnesde151cf2008-11-12 10:03:55 -08002545 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
Eric Anholt4e901fd2009-10-26 16:44:17 -07002546 } else {
Eric Anholtdc529a42009-03-10 22:34:49 -07002547 uint32_t fence_reg;
2548
2549 if (obj_priv->fence_reg < 8)
2550 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
2551 else
2552 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg -
2553 8) * 4;
2554
2555 I915_WRITE(fence_reg, 0);
2556 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08002557
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002558 reg->obj = NULL;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002559 obj_priv->fence_reg = I915_FENCE_REG_NONE;
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002560 list_del_init(&reg->lru_list);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002561}
2562
Eric Anholt673a3942008-07-30 12:06:12 -07002563/**
Chris Wilson52dc7d32009-06-06 09:46:01 +01002564 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2565 * to the buffer to finish, and then resets the fence register.
2566 * @obj: tiled object holding a fence register.
2567 *
2568 * Zeroes out the fence register itself and clears out the associated
2569 * data structures in dev_priv and obj_priv.
2570 */
2571int
2572i915_gem_object_put_fence_reg(struct drm_gem_object *obj)
2573{
2574 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01002575 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson52dc7d32009-06-06 09:46:01 +01002576
2577 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2578 return 0;
2579
Daniel Vetter10ae9bd2010-02-01 13:59:17 +01002580 /* If we've changed tiling, GTT-mappings of the object
2581 * need to re-fault to ensure that the correct fence register
2582 * setup is in place.
2583 */
2584 i915_gem_release_mmap(obj);
2585
Chris Wilson52dc7d32009-06-06 09:46:01 +01002586 /* On the i915, GPU access to tiled buffers is via a fence,
2587 * therefore we must wait for any outstanding access to complete
2588 * before clearing the fence.
2589 */
2590 if (!IS_I965G(dev)) {
2591 int ret;
2592
2593 i915_gem_object_flush_gpu_write_domain(obj);
Chris Wilson52dc7d32009-06-06 09:46:01 +01002594 ret = i915_gem_object_wait_rendering(obj);
2595 if (ret != 0)
2596 return ret;
2597 }
2598
Daniel Vetter4a726612010-02-01 13:59:16 +01002599 i915_gem_object_flush_gtt_write_domain(obj);
Chris Wilson52dc7d32009-06-06 09:46:01 +01002600 i915_gem_clear_fence_reg (obj);
2601
2602 return 0;
2603}
2604
2605/**
Eric Anholt673a3942008-07-30 12:06:12 -07002606 * Finds free space in the GTT aperture and binds the object there.
2607 */
2608static int
2609i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2610{
2611 struct drm_device *dev = obj->dev;
2612 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002613 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002614 struct drm_mm_node *free_space;
Chris Wilson4bdadb92010-01-27 13:36:32 +00002615 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
Chris Wilson07f73f62009-09-14 16:50:30 +01002616 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002617
Chris Wilsonbb6baf72009-09-22 14:24:13 +01002618 if (obj_priv->madv != I915_MADV_WILLNEED) {
Chris Wilson3ef94da2009-09-14 16:50:29 +01002619 DRM_ERROR("Attempting to bind a purgeable object\n");
2620 return -EINVAL;
2621 }
2622
Eric Anholt673a3942008-07-30 12:06:12 -07002623 if (alignment == 0)
Jesse Barnes0f973f22009-01-26 17:10:45 -08002624 alignment = i915_gem_get_gtt_alignment(obj);
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002625 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
Eric Anholt673a3942008-07-30 12:06:12 -07002626 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2627 return -EINVAL;
2628 }
2629
Chris Wilson654fc602010-05-27 13:18:21 +01002630 /* If the object is bigger than the entire aperture, reject it early
2631 * before evicting everything in a vain attempt to find space.
2632 */
2633 if (obj->size > dev->gtt_total) {
2634 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2635 return -E2BIG;
2636 }
2637
Eric Anholt673a3942008-07-30 12:06:12 -07002638 search_free:
2639 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2640 obj->size, alignment, 0);
2641 if (free_space != NULL) {
2642 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2643 alignment);
2644 if (obj_priv->gtt_space != NULL) {
2645 obj_priv->gtt_space->private = obj;
2646 obj_priv->gtt_offset = obj_priv->gtt_space->start;
2647 }
2648 }
2649 if (obj_priv->gtt_space == NULL) {
2650 /* If the gtt is empty and we're still having trouble
2651 * fitting our object in, we're out of memory.
2652 */
2653#if WATCH_LRU
2654 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2655#endif
Chris Wilson07f73f62009-09-14 16:50:30 +01002656 ret = i915_gem_evict_something(dev, obj->size);
Chris Wilson97311292009-09-21 00:22:34 +01002657 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07002658 return ret;
Chris Wilson97311292009-09-21 00:22:34 +01002659
Eric Anholt673a3942008-07-30 12:06:12 -07002660 goto search_free;
2661 }
2662
2663#if WATCH_BUF
Krzysztof Halasacfd43c02009-06-20 00:31:28 +02002664 DRM_INFO("Binding object of size %zd at 0x%08x\n",
Eric Anholt673a3942008-07-30 12:06:12 -07002665 obj->size, obj_priv->gtt_offset);
2666#endif
Chris Wilson4bdadb92010-01-27 13:36:32 +00002667 ret = i915_gem_object_get_pages(obj, gfpmask);
Eric Anholt673a3942008-07-30 12:06:12 -07002668 if (ret) {
2669 drm_mm_put_block(obj_priv->gtt_space);
2670 obj_priv->gtt_space = NULL;
Chris Wilson07f73f62009-09-14 16:50:30 +01002671
2672 if (ret == -ENOMEM) {
2673 /* first try to clear up some space from the GTT */
2674 ret = i915_gem_evict_something(dev, obj->size);
2675 if (ret) {
Chris Wilson07f73f62009-09-14 16:50:30 +01002676 /* now try to shrink everyone else */
Chris Wilson4bdadb92010-01-27 13:36:32 +00002677 if (gfpmask) {
2678 gfpmask = 0;
2679 goto search_free;
Chris Wilson07f73f62009-09-14 16:50:30 +01002680 }
2681
2682 return ret;
2683 }
2684
2685 goto search_free;
2686 }
2687
Eric Anholt673a3942008-07-30 12:06:12 -07002688 return ret;
2689 }
2690
Eric Anholt673a3942008-07-30 12:06:12 -07002691 /* Create an AGP memory structure pointing at our pages, and bind it
2692 * into the GTT.
2693 */
2694 obj_priv->agp_mem = drm_agp_bind_pages(dev,
Eric Anholt856fa192009-03-19 14:10:50 -07002695 obj_priv->pages,
Chris Wilson07f73f62009-09-14 16:50:30 +01002696 obj->size >> PAGE_SHIFT,
Keith Packardba1eb1d2008-10-14 19:55:10 -07002697 obj_priv->gtt_offset,
2698 obj_priv->agp_type);
Eric Anholt673a3942008-07-30 12:06:12 -07002699 if (obj_priv->agp_mem == NULL) {
Eric Anholt856fa192009-03-19 14:10:50 -07002700 i915_gem_object_put_pages(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002701 drm_mm_put_block(obj_priv->gtt_space);
2702 obj_priv->gtt_space = NULL;
Chris Wilson07f73f62009-09-14 16:50:30 +01002703
2704 ret = i915_gem_evict_something(dev, obj->size);
Chris Wilson97311292009-09-21 00:22:34 +01002705 if (ret)
Chris Wilson07f73f62009-09-14 16:50:30 +01002706 return ret;
Chris Wilson07f73f62009-09-14 16:50:30 +01002707
2708 goto search_free;
Eric Anholt673a3942008-07-30 12:06:12 -07002709 }
2710 atomic_inc(&dev->gtt_count);
2711 atomic_add(obj->size, &dev->gtt_memory);
2712
2713 /* Assert that the object is not currently in any GPU domain. As it
2714 * wasn't in the GTT, there shouldn't be any way it could have been in
2715 * a GPU cache
2716 */
Chris Wilson21d509e2009-06-06 09:46:02 +01002717 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2718 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
Eric Anholt673a3942008-07-30 12:06:12 -07002719
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002720 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2721
Eric Anholt673a3942008-07-30 12:06:12 -07002722 return 0;
2723}
2724
2725void
2726i915_gem_clflush_object(struct drm_gem_object *obj)
2727{
Daniel Vetter23010e42010-03-08 13:35:02 +01002728 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002729
2730 /* If we don't have a page list set up, then we're not pinned
2731 * to GPU, and we can ignore the cache flush because it'll happen
2732 * again at bind time.
2733 */
Eric Anholt856fa192009-03-19 14:10:50 -07002734 if (obj_priv->pages == NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07002735 return;
2736
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002737 trace_i915_gem_object_clflush(obj);
Eric Anholtcfa16a02009-05-26 18:46:16 -07002738
Eric Anholt856fa192009-03-19 14:10:50 -07002739 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
Eric Anholt673a3942008-07-30 12:06:12 -07002740}
2741
Eric Anholte47c68e2008-11-14 13:35:19 -08002742/** Flushes any GPU write domain for the object if it's dirty. */
2743static void
2744i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
2745{
2746 struct drm_device *dev = obj->dev;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002747 uint32_t old_write_domain;
Zou Nan hai852835f2010-05-21 09:08:56 +08002748 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08002749
2750 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
2751 return;
2752
2753 /* Queue the GPU write cache flushing we need. */
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002754 old_write_domain = obj->write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002755 i915_gem_flush(dev, 0, obj->write_domain);
Zou Nan hai852835f2010-05-21 09:08:56 +08002756 (void) i915_add_request(dev, NULL, obj->write_domain, obj_priv->ring);
Daniel Vetter99fcb762010-02-07 16:20:18 +01002757 BUG_ON(obj->write_domain);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002758
2759 trace_i915_gem_object_change_domain(obj,
2760 obj->read_domains,
2761 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08002762}
2763
2764/** Flushes the GTT write domain for the object if it's dirty. */
2765static void
2766i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2767{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002768 uint32_t old_write_domain;
2769
Eric Anholte47c68e2008-11-14 13:35:19 -08002770 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2771 return;
2772
2773 /* No actual flushing is required for the GTT write domain. Writes
2774 * to it immediately go to main memory as far as we know, so there's
2775 * no chipset flush. It also doesn't land in render cache.
2776 */
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002777 old_write_domain = obj->write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002778 obj->write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002779
2780 trace_i915_gem_object_change_domain(obj,
2781 obj->read_domains,
2782 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08002783}
2784
2785/** Flushes the CPU write domain for the object if it's dirty. */
2786static void
2787i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2788{
2789 struct drm_device *dev = obj->dev;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002790 uint32_t old_write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002791
2792 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2793 return;
2794
2795 i915_gem_clflush_object(obj);
2796 drm_agp_chipset_flush(dev);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002797 old_write_domain = obj->write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002798 obj->write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002799
2800 trace_i915_gem_object_change_domain(obj,
2801 obj->read_domains,
2802 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08002803}
2804
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05002805void
2806i915_gem_object_flush_write_domain(struct drm_gem_object *obj)
2807{
2808 switch (obj->write_domain) {
2809 case I915_GEM_DOMAIN_GTT:
2810 i915_gem_object_flush_gtt_write_domain(obj);
2811 break;
2812 case I915_GEM_DOMAIN_CPU:
2813 i915_gem_object_flush_cpu_write_domain(obj);
2814 break;
2815 default:
2816 i915_gem_object_flush_gpu_write_domain(obj);
2817 break;
2818 }
2819}
2820
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002821/**
2822 * Moves a single object to the GTT read, and possibly write domain.
2823 *
2824 * This function returns when the move is complete, including waiting on
2825 * flushes to occur.
2826 */
Jesse Barnes79e53942008-11-07 14:24:08 -08002827int
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002828i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2829{
Daniel Vetter23010e42010-03-08 13:35:02 +01002830 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002831 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08002832 int ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002833
Eric Anholt02354392008-11-26 13:58:13 -08002834 /* Not valid to be called on unbound objects. */
2835 if (obj_priv->gtt_space == NULL)
2836 return -EINVAL;
2837
Eric Anholte47c68e2008-11-14 13:35:19 -08002838 i915_gem_object_flush_gpu_write_domain(obj);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002839 /* Wait on any GPU rendering and flushing to occur. */
Eric Anholte47c68e2008-11-14 13:35:19 -08002840 ret = i915_gem_object_wait_rendering(obj);
2841 if (ret != 0)
2842 return ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002843
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002844 old_write_domain = obj->write_domain;
2845 old_read_domains = obj->read_domains;
2846
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002847 /* If we're writing through the GTT domain, then CPU and GPU caches
2848 * will need to be invalidated at next use.
2849 */
2850 if (write)
Eric Anholte47c68e2008-11-14 13:35:19 -08002851 obj->read_domains &= I915_GEM_DOMAIN_GTT;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002852
Eric Anholte47c68e2008-11-14 13:35:19 -08002853 i915_gem_object_flush_cpu_write_domain(obj);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002854
2855 /* It should now be out of any other write domains, and we can update
2856 * the domain values for our changes.
2857 */
2858 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2859 obj->read_domains |= I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08002860 if (write) {
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002861 obj->write_domain = I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08002862 obj_priv->dirty = 1;
2863 }
2864
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002865 trace_i915_gem_object_change_domain(obj,
2866 old_read_domains,
2867 old_write_domain);
2868
Eric Anholte47c68e2008-11-14 13:35:19 -08002869 return 0;
2870}
2871
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002872/*
2873 * Prepare buffer for display plane. Use uninterruptible for possible flush
2874 * wait, as in modesetting process we're not supposed to be interrupted.
2875 */
2876int
2877i915_gem_object_set_to_display_plane(struct drm_gem_object *obj)
2878{
2879 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01002880 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002881 uint32_t old_write_domain, old_read_domains;
2882 int ret;
2883
2884 /* Not valid to be called on unbound objects. */
2885 if (obj_priv->gtt_space == NULL)
2886 return -EINVAL;
2887
2888 i915_gem_object_flush_gpu_write_domain(obj);
2889
2890 /* Wait on any GPU rendering and flushing to occur. */
2891 if (obj_priv->active) {
2892#if WATCH_BUF
2893 DRM_INFO("%s: object %p wait for seqno %08x\n",
2894 __func__, obj, obj_priv->last_rendering_seqno);
2895#endif
Zou Nan hai852835f2010-05-21 09:08:56 +08002896 ret = i915_do_wait_request(dev,
2897 obj_priv->last_rendering_seqno,
2898 0,
2899 obj_priv->ring);
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002900 if (ret != 0)
2901 return ret;
2902 }
2903
Chris Wilsonb118c1e2010-05-27 13:18:14 +01002904 i915_gem_object_flush_cpu_write_domain(obj);
2905
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002906 old_write_domain = obj->write_domain;
2907 old_read_domains = obj->read_domains;
2908
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002909 /* It should now be out of any other write domains, and we can update
2910 * the domain values for our changes.
2911 */
2912 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
Chris Wilsonb118c1e2010-05-27 13:18:14 +01002913 obj->read_domains = I915_GEM_DOMAIN_GTT;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002914 obj->write_domain = I915_GEM_DOMAIN_GTT;
2915 obj_priv->dirty = 1;
2916
2917 trace_i915_gem_object_change_domain(obj,
2918 old_read_domains,
2919 old_write_domain);
2920
2921 return 0;
2922}
2923
Eric Anholte47c68e2008-11-14 13:35:19 -08002924/**
2925 * Moves a single object to the CPU read, and possibly write domain.
2926 *
2927 * This function returns when the move is complete, including waiting on
2928 * flushes to occur.
2929 */
2930static int
2931i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2932{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002933 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08002934 int ret;
2935
2936 i915_gem_object_flush_gpu_write_domain(obj);
2937 /* Wait on any GPU rendering and flushing to occur. */
2938 ret = i915_gem_object_wait_rendering(obj);
2939 if (ret != 0)
2940 return ret;
2941
2942 i915_gem_object_flush_gtt_write_domain(obj);
2943
2944 /* If we have a partially-valid cache of the object in the CPU,
2945 * finish invalidating it and free the per-page flags.
2946 */
2947 i915_gem_object_set_to_full_cpu_read_domain(obj);
2948
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002949 old_write_domain = obj->write_domain;
2950 old_read_domains = obj->read_domains;
2951
Eric Anholte47c68e2008-11-14 13:35:19 -08002952 /* Flush the CPU cache if it's still invalid. */
2953 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2954 i915_gem_clflush_object(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08002955
2956 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2957 }
2958
2959 /* It should now be out of any other write domains, and we can update
2960 * the domain values for our changes.
2961 */
2962 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2963
2964 /* If we're writing through the CPU, then the GPU read domains will
2965 * need to be invalidated at next use.
2966 */
2967 if (write) {
2968 obj->read_domains &= I915_GEM_DOMAIN_CPU;
2969 obj->write_domain = I915_GEM_DOMAIN_CPU;
2970 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002971
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002972 trace_i915_gem_object_change_domain(obj,
2973 old_read_domains,
2974 old_write_domain);
2975
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002976 return 0;
2977}
2978
Eric Anholt673a3942008-07-30 12:06:12 -07002979/*
2980 * Set the next domain for the specified object. This
2981 * may not actually perform the necessary flushing/invaliding though,
2982 * as that may want to be batched with other set_domain operations
2983 *
2984 * This is (we hope) the only really tricky part of gem. The goal
2985 * is fairly simple -- track which caches hold bits of the object
2986 * and make sure they remain coherent. A few concrete examples may
2987 * help to explain how it works. For shorthand, we use the notation
2988 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2989 * a pair of read and write domain masks.
2990 *
2991 * Case 1: the batch buffer
2992 *
2993 * 1. Allocated
2994 * 2. Written by CPU
2995 * 3. Mapped to GTT
2996 * 4. Read by GPU
2997 * 5. Unmapped from GTT
2998 * 6. Freed
2999 *
3000 * Let's take these a step at a time
3001 *
3002 * 1. Allocated
3003 * Pages allocated from the kernel may still have
3004 * cache contents, so we set them to (CPU, CPU) always.
3005 * 2. Written by CPU (using pwrite)
3006 * The pwrite function calls set_domain (CPU, CPU) and
3007 * this function does nothing (as nothing changes)
3008 * 3. Mapped by GTT
3009 * This function asserts that the object is not
3010 * currently in any GPU-based read or write domains
3011 * 4. Read by GPU
3012 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
3013 * As write_domain is zero, this function adds in the
3014 * current read domains (CPU+COMMAND, 0).
3015 * flush_domains is set to CPU.
3016 * invalidate_domains is set to COMMAND
3017 * clflush is run to get data out of the CPU caches
3018 * then i915_dev_set_domain calls i915_gem_flush to
3019 * emit an MI_FLUSH and drm_agp_chipset_flush
3020 * 5. Unmapped from GTT
3021 * i915_gem_object_unbind calls set_domain (CPU, CPU)
3022 * flush_domains and invalidate_domains end up both zero
3023 * so no flushing/invalidating happens
3024 * 6. Freed
3025 * yay, done
3026 *
3027 * Case 2: The shared render buffer
3028 *
3029 * 1. Allocated
3030 * 2. Mapped to GTT
3031 * 3. Read/written by GPU
3032 * 4. set_domain to (CPU,CPU)
3033 * 5. Read/written by CPU
3034 * 6. Read/written by GPU
3035 *
3036 * 1. Allocated
3037 * Same as last example, (CPU, CPU)
3038 * 2. Mapped to GTT
3039 * Nothing changes (assertions find that it is not in the GPU)
3040 * 3. Read/written by GPU
3041 * execbuffer calls set_domain (RENDER, RENDER)
3042 * flush_domains gets CPU
3043 * invalidate_domains gets GPU
3044 * clflush (obj)
3045 * MI_FLUSH and drm_agp_chipset_flush
3046 * 4. set_domain (CPU, CPU)
3047 * flush_domains gets GPU
3048 * invalidate_domains gets CPU
3049 * wait_rendering (obj) to make sure all drawing is complete.
3050 * This will include an MI_FLUSH to get the data from GPU
3051 * to memory
3052 * clflush (obj) to invalidate the CPU cache
3053 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
3054 * 5. Read/written by CPU
3055 * cache lines are loaded and dirtied
3056 * 6. Read written by GPU
3057 * Same as last GPU access
3058 *
3059 * Case 3: The constant buffer
3060 *
3061 * 1. Allocated
3062 * 2. Written by CPU
3063 * 3. Read by GPU
3064 * 4. Updated (written) by CPU again
3065 * 5. Read by GPU
3066 *
3067 * 1. Allocated
3068 * (CPU, CPU)
3069 * 2. Written by CPU
3070 * (CPU, CPU)
3071 * 3. Read by GPU
3072 * (CPU+RENDER, 0)
3073 * flush_domains = CPU
3074 * invalidate_domains = RENDER
3075 * clflush (obj)
3076 * MI_FLUSH
3077 * drm_agp_chipset_flush
3078 * 4. Updated (written) by CPU again
3079 * (CPU, CPU)
3080 * flush_domains = 0 (no previous write domain)
3081 * invalidate_domains = 0 (no new read domains)
3082 * 5. Read by GPU
3083 * (CPU+RENDER, 0)
3084 * flush_domains = CPU
3085 * invalidate_domains = RENDER
3086 * clflush (obj)
3087 * MI_FLUSH
3088 * drm_agp_chipset_flush
3089 */
Keith Packardc0d90822008-11-20 23:11:08 -08003090static void
Eric Anholt8b0e3782009-02-19 14:40:50 -08003091i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07003092{
3093 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01003094 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003095 uint32_t invalidate_domains = 0;
3096 uint32_t flush_domains = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003097 uint32_t old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003098
Eric Anholt8b0e3782009-02-19 14:40:50 -08003099 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3100 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
Eric Anholt673a3942008-07-30 12:06:12 -07003101
Jesse Barnes652c3932009-08-17 13:31:43 -07003102 intel_mark_busy(dev, obj);
3103
Eric Anholt673a3942008-07-30 12:06:12 -07003104#if WATCH_BUF
3105 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
3106 __func__, obj,
Eric Anholt8b0e3782009-02-19 14:40:50 -08003107 obj->read_domains, obj->pending_read_domains,
3108 obj->write_domain, obj->pending_write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07003109#endif
3110 /*
3111 * If the object isn't moving to a new write domain,
3112 * let the object stay in multiple read domains
3113 */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003114 if (obj->pending_write_domain == 0)
3115 obj->pending_read_domains |= obj->read_domains;
Eric Anholt673a3942008-07-30 12:06:12 -07003116 else
3117 obj_priv->dirty = 1;
3118
3119 /*
3120 * Flush the current write domain if
3121 * the new read domains don't match. Invalidate
3122 * any read domains which differ from the old
3123 * write domain
3124 */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003125 if (obj->write_domain &&
3126 obj->write_domain != obj->pending_read_domains) {
Eric Anholt673a3942008-07-30 12:06:12 -07003127 flush_domains |= obj->write_domain;
Eric Anholt8b0e3782009-02-19 14:40:50 -08003128 invalidate_domains |=
3129 obj->pending_read_domains & ~obj->write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -07003130 }
3131 /*
3132 * Invalidate any read caches which may have
3133 * stale data. That is, any new read domains.
3134 */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003135 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
Eric Anholt673a3942008-07-30 12:06:12 -07003136 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3137#if WATCH_BUF
3138 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3139 __func__, flush_domains, invalidate_domains);
3140#endif
Eric Anholt673a3942008-07-30 12:06:12 -07003141 i915_gem_clflush_object(obj);
3142 }
3143
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003144 old_read_domains = obj->read_domains;
3145
Eric Anholtefbeed92009-02-19 14:54:51 -08003146 /* The actual obj->write_domain will be updated with
3147 * pending_write_domain after we emit the accumulated flush for all
3148 * of our domain changes in execbuffers (which clears objects'
3149 * write_domains). So if we have a current write domain that we
3150 * aren't changing, set pending_write_domain to that.
3151 */
3152 if (flush_domains == 0 && obj->pending_write_domain == 0)
3153 obj->pending_write_domain = obj->write_domain;
Eric Anholt8b0e3782009-02-19 14:40:50 -08003154 obj->read_domains = obj->pending_read_domains;
Eric Anholt673a3942008-07-30 12:06:12 -07003155
3156 dev->invalidate_domains |= invalidate_domains;
3157 dev->flush_domains |= flush_domains;
3158#if WATCH_BUF
3159 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3160 __func__,
3161 obj->read_domains, obj->write_domain,
3162 dev->invalidate_domains, dev->flush_domains);
3163#endif
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003164
3165 trace_i915_gem_object_change_domain(obj,
3166 old_read_domains,
3167 obj->write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07003168}
3169
3170/**
Eric Anholte47c68e2008-11-14 13:35:19 -08003171 * Moves the object from a partially CPU read to a full one.
Eric Anholt673a3942008-07-30 12:06:12 -07003172 *
Eric Anholte47c68e2008-11-14 13:35:19 -08003173 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3174 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3175 */
3176static void
3177i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3178{
Daniel Vetter23010e42010-03-08 13:35:02 +01003179 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08003180
3181 if (!obj_priv->page_cpu_valid)
3182 return;
3183
3184 /* If we're partially in the CPU read domain, finish moving it in.
3185 */
3186 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3187 int i;
3188
3189 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3190 if (obj_priv->page_cpu_valid[i])
3191 continue;
Eric Anholt856fa192009-03-19 14:10:50 -07003192 drm_clflush_pages(obj_priv->pages + i, 1);
Eric Anholte47c68e2008-11-14 13:35:19 -08003193 }
Eric Anholte47c68e2008-11-14 13:35:19 -08003194 }
3195
3196 /* Free the page_cpu_valid mappings which are now stale, whether
3197 * or not we've got I915_GEM_DOMAIN_CPU.
3198 */
Eric Anholt9a298b22009-03-24 12:23:04 -07003199 kfree(obj_priv->page_cpu_valid);
Eric Anholte47c68e2008-11-14 13:35:19 -08003200 obj_priv->page_cpu_valid = NULL;
3201}
3202
3203/**
3204 * Set the CPU read domain on a range of the object.
3205 *
3206 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3207 * not entirely valid. The page_cpu_valid member of the object flags which
3208 * pages have been flushed, and will be respected by
3209 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3210 * of the whole object.
3211 *
3212 * This function returns when the move is complete, including waiting on
3213 * flushes to occur.
Eric Anholt673a3942008-07-30 12:06:12 -07003214 */
3215static int
Eric Anholte47c68e2008-11-14 13:35:19 -08003216i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3217 uint64_t offset, uint64_t size)
Eric Anholt673a3942008-07-30 12:06:12 -07003218{
Daniel Vetter23010e42010-03-08 13:35:02 +01003219 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003220 uint32_t old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003221 int i, ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003222
Eric Anholte47c68e2008-11-14 13:35:19 -08003223 if (offset == 0 && size == obj->size)
3224 return i915_gem_object_set_to_cpu_domain(obj, 0);
3225
3226 i915_gem_object_flush_gpu_write_domain(obj);
3227 /* Wait on any GPU rendering and flushing to occur. */
3228 ret = i915_gem_object_wait_rendering(obj);
3229 if (ret != 0)
3230 return ret;
3231 i915_gem_object_flush_gtt_write_domain(obj);
3232
3233 /* If we're already fully in the CPU read domain, we're done. */
3234 if (obj_priv->page_cpu_valid == NULL &&
3235 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07003236 return 0;
3237
Eric Anholte47c68e2008-11-14 13:35:19 -08003238 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3239 * newly adding I915_GEM_DOMAIN_CPU
3240 */
Eric Anholt673a3942008-07-30 12:06:12 -07003241 if (obj_priv->page_cpu_valid == NULL) {
Eric Anholt9a298b22009-03-24 12:23:04 -07003242 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3243 GFP_KERNEL);
Eric Anholte47c68e2008-11-14 13:35:19 -08003244 if (obj_priv->page_cpu_valid == NULL)
3245 return -ENOMEM;
3246 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3247 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
Eric Anholt673a3942008-07-30 12:06:12 -07003248
3249 /* Flush the cache on any pages that are still invalid from the CPU's
3250 * perspective.
3251 */
Eric Anholte47c68e2008-11-14 13:35:19 -08003252 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3253 i++) {
Eric Anholt673a3942008-07-30 12:06:12 -07003254 if (obj_priv->page_cpu_valid[i])
3255 continue;
3256
Eric Anholt856fa192009-03-19 14:10:50 -07003257 drm_clflush_pages(obj_priv->pages + i, 1);
Eric Anholt673a3942008-07-30 12:06:12 -07003258
3259 obj_priv->page_cpu_valid[i] = 1;
3260 }
3261
Eric Anholte47c68e2008-11-14 13:35:19 -08003262 /* It should now be out of any other write domains, and we can update
3263 * the domain values for our changes.
3264 */
3265 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3266
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003267 old_read_domains = obj->read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003268 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3269
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003270 trace_i915_gem_object_change_domain(obj,
3271 old_read_domains,
3272 obj->write_domain);
3273
Eric Anholt673a3942008-07-30 12:06:12 -07003274 return 0;
3275}
3276
3277/**
Eric Anholt673a3942008-07-30 12:06:12 -07003278 * Pin an object to the GTT and evaluate the relocations landing in it.
3279 */
3280static int
3281i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3282 struct drm_file *file_priv,
Jesse Barnes76446ca2009-12-17 22:05:42 -05003283 struct drm_i915_gem_exec_object2 *entry,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003284 struct drm_i915_gem_relocation_entry *relocs)
Eric Anholt673a3942008-07-30 12:06:12 -07003285{
3286 struct drm_device *dev = obj->dev;
Keith Packard0839ccb2008-10-30 19:38:48 -07003287 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01003288 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003289 int i, ret;
Keith Packard0839ccb2008-10-30 19:38:48 -07003290 void __iomem *reloc_page;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003291 bool need_fence;
3292
3293 need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3294 obj_priv->tiling_mode != I915_TILING_NONE;
3295
3296 /* Check fence reg constraints and rebind if necessary */
Chris Wilson808b24d2010-05-27 13:18:15 +01003297 if (need_fence &&
3298 !i915_gem_object_fence_offset_ok(obj,
3299 obj_priv->tiling_mode)) {
3300 ret = i915_gem_object_unbind(obj);
3301 if (ret)
3302 return ret;
3303 }
Eric Anholt673a3942008-07-30 12:06:12 -07003304
3305 /* Choose the GTT offset for our buffer and put it there. */
3306 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3307 if (ret)
3308 return ret;
3309
Jesse Barnes76446ca2009-12-17 22:05:42 -05003310 /*
3311 * Pre-965 chips need a fence register set up in order to
3312 * properly handle blits to/from tiled surfaces.
3313 */
3314 if (need_fence) {
3315 ret = i915_gem_object_get_fence_reg(obj);
3316 if (ret != 0) {
Jesse Barnes76446ca2009-12-17 22:05:42 -05003317 i915_gem_object_unpin(obj);
3318 return ret;
3319 }
3320 }
3321
Eric Anholt673a3942008-07-30 12:06:12 -07003322 entry->offset = obj_priv->gtt_offset;
3323
Eric Anholt673a3942008-07-30 12:06:12 -07003324 /* Apply the relocations, using the GTT aperture to avoid cache
3325 * flushing requirements.
3326 */
3327 for (i = 0; i < entry->relocation_count; i++) {
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003328 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
Eric Anholt673a3942008-07-30 12:06:12 -07003329 struct drm_gem_object *target_obj;
3330 struct drm_i915_gem_object *target_obj_priv;
Eric Anholt3043c602008-10-02 12:24:47 -07003331 uint32_t reloc_val, reloc_offset;
3332 uint32_t __iomem *reloc_entry;
Eric Anholt673a3942008-07-30 12:06:12 -07003333
Eric Anholt673a3942008-07-30 12:06:12 -07003334 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003335 reloc->target_handle);
Eric Anholt673a3942008-07-30 12:06:12 -07003336 if (target_obj == NULL) {
3337 i915_gem_object_unpin(obj);
3338 return -EBADF;
3339 }
Daniel Vetter23010e42010-03-08 13:35:02 +01003340 target_obj_priv = to_intel_bo(target_obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003341
Chris Wilson8542a0b2009-09-09 21:15:15 +01003342#if WATCH_RELOC
3343 DRM_INFO("%s: obj %p offset %08x target %d "
3344 "read %08x write %08x gtt %08x "
3345 "presumed %08x delta %08x\n",
3346 __func__,
3347 obj,
3348 (int) reloc->offset,
3349 (int) reloc->target_handle,
3350 (int) reloc->read_domains,
3351 (int) reloc->write_domain,
3352 (int) target_obj_priv->gtt_offset,
3353 (int) reloc->presumed_offset,
3354 reloc->delta);
3355#endif
3356
Eric Anholt673a3942008-07-30 12:06:12 -07003357 /* The target buffer should have appeared before us in the
3358 * exec_object list, so it should have a GTT space bound by now.
3359 */
3360 if (target_obj_priv->gtt_space == NULL) {
3361 DRM_ERROR("No GTT space found for object %d\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003362 reloc->target_handle);
Eric Anholt673a3942008-07-30 12:06:12 -07003363 drm_gem_object_unreference(target_obj);
3364 i915_gem_object_unpin(obj);
3365 return -EINVAL;
3366 }
3367
Chris Wilson8542a0b2009-09-09 21:15:15 +01003368 /* Validate that the target is in a valid r/w GPU domain */
Daniel Vetter16edd552010-02-19 11:52:02 +01003369 if (reloc->write_domain & (reloc->write_domain - 1)) {
3370 DRM_ERROR("reloc with multiple write domains: "
3371 "obj %p target %d offset %d "
3372 "read %08x write %08x",
3373 obj, reloc->target_handle,
3374 (int) reloc->offset,
3375 reloc->read_domains,
3376 reloc->write_domain);
3377 return -EINVAL;
3378 }
Chris Wilson8542a0b2009-09-09 21:15:15 +01003379 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3380 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3381 DRM_ERROR("reloc with read/write CPU domains: "
3382 "obj %p target %d offset %d "
3383 "read %08x write %08x",
3384 obj, reloc->target_handle,
3385 (int) reloc->offset,
3386 reloc->read_domains,
3387 reloc->write_domain);
3388 drm_gem_object_unreference(target_obj);
3389 i915_gem_object_unpin(obj);
3390 return -EINVAL;
3391 }
3392 if (reloc->write_domain && target_obj->pending_write_domain &&
3393 reloc->write_domain != target_obj->pending_write_domain) {
3394 DRM_ERROR("Write domain conflict: "
3395 "obj %p target %d offset %d "
3396 "new %08x old %08x\n",
3397 obj, reloc->target_handle,
3398 (int) reloc->offset,
3399 reloc->write_domain,
3400 target_obj->pending_write_domain);
3401 drm_gem_object_unreference(target_obj);
3402 i915_gem_object_unpin(obj);
3403 return -EINVAL;
3404 }
3405
3406 target_obj->pending_read_domains |= reloc->read_domains;
3407 target_obj->pending_write_domain |= reloc->write_domain;
3408
3409 /* If the relocation already has the right value in it, no
3410 * more work needs to be done.
3411 */
3412 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3413 drm_gem_object_unreference(target_obj);
3414 continue;
3415 }
3416
3417 /* Check that the relocation address is valid... */
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003418 if (reloc->offset > obj->size - 4) {
Eric Anholt673a3942008-07-30 12:06:12 -07003419 DRM_ERROR("Relocation beyond object bounds: "
3420 "obj %p target %d offset %d size %d.\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003421 obj, reloc->target_handle,
3422 (int) reloc->offset, (int) obj->size);
Eric Anholt673a3942008-07-30 12:06:12 -07003423 drm_gem_object_unreference(target_obj);
3424 i915_gem_object_unpin(obj);
3425 return -EINVAL;
3426 }
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003427 if (reloc->offset & 3) {
Eric Anholt673a3942008-07-30 12:06:12 -07003428 DRM_ERROR("Relocation not 4-byte aligned: "
3429 "obj %p target %d offset %d.\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003430 obj, reloc->target_handle,
3431 (int) reloc->offset);
Eric Anholt673a3942008-07-30 12:06:12 -07003432 drm_gem_object_unreference(target_obj);
3433 i915_gem_object_unpin(obj);
3434 return -EINVAL;
3435 }
3436
Chris Wilson8542a0b2009-09-09 21:15:15 +01003437 /* and points to somewhere within the target object. */
Chris Wilsoncd0b9fb2009-09-15 23:23:18 +01003438 if (reloc->delta >= target_obj->size) {
3439 DRM_ERROR("Relocation beyond target object bounds: "
3440 "obj %p target %d delta %d size %d.\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003441 obj, reloc->target_handle,
Chris Wilsoncd0b9fb2009-09-15 23:23:18 +01003442 (int) reloc->delta, (int) target_obj->size);
Chris Wilson491152b2009-02-11 14:26:32 +00003443 drm_gem_object_unreference(target_obj);
3444 i915_gem_object_unpin(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08003445 return -EINVAL;
3446 }
3447
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003448 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3449 if (ret != 0) {
3450 drm_gem_object_unreference(target_obj);
3451 i915_gem_object_unpin(obj);
3452 return -EINVAL;
Eric Anholt673a3942008-07-30 12:06:12 -07003453 }
3454
3455 /* Map the page containing the relocation we're going to
3456 * perform.
3457 */
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003458 reloc_offset = obj_priv->gtt_offset + reloc->offset;
Keith Packard0839ccb2008-10-30 19:38:48 -07003459 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3460 (reloc_offset &
3461 ~(PAGE_SIZE - 1)));
Eric Anholt3043c602008-10-02 12:24:47 -07003462 reloc_entry = (uint32_t __iomem *)(reloc_page +
Keith Packard0839ccb2008-10-30 19:38:48 -07003463 (reloc_offset & (PAGE_SIZE - 1)));
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003464 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
Eric Anholt673a3942008-07-30 12:06:12 -07003465
3466#if WATCH_BUF
3467 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003468 obj, (unsigned int) reloc->offset,
Eric Anholt673a3942008-07-30 12:06:12 -07003469 readl(reloc_entry), reloc_val);
3470#endif
3471 writel(reloc_val, reloc_entry);
Keith Packard0839ccb2008-10-30 19:38:48 -07003472 io_mapping_unmap_atomic(reloc_page);
Eric Anholt673a3942008-07-30 12:06:12 -07003473
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003474 /* The updated presumed offset for this entry will be
3475 * copied back out to the user.
Eric Anholt673a3942008-07-30 12:06:12 -07003476 */
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003477 reloc->presumed_offset = target_obj_priv->gtt_offset;
Eric Anholt673a3942008-07-30 12:06:12 -07003478
3479 drm_gem_object_unreference(target_obj);
3480 }
3481
Eric Anholt673a3942008-07-30 12:06:12 -07003482#if WATCH_BUF
3483 if (0)
3484 i915_gem_dump_object(obj, 128, __func__, ~0);
3485#endif
3486 return 0;
3487}
3488
Eric Anholt673a3942008-07-30 12:06:12 -07003489/* Throttle our rendering by waiting until the ring has completed our requests
3490 * emitted over 20 msec ago.
3491 *
Eric Anholtb9624422009-06-03 07:27:35 +00003492 * Note that if we were to use the current jiffies each time around the loop,
3493 * we wouldn't escape the function with any frames outstanding if the time to
3494 * render a frame was over 20ms.
3495 *
Eric Anholt673a3942008-07-30 12:06:12 -07003496 * This should get us reasonable parallelism between CPU and GPU but also
3497 * relatively low latency when blocking on a particular request to finish.
3498 */
3499static int
3500i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3501{
3502 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3503 int ret = 0;
Eric Anholtb9624422009-06-03 07:27:35 +00003504 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
Eric Anholt673a3942008-07-30 12:06:12 -07003505
3506 mutex_lock(&dev->struct_mutex);
Eric Anholtb9624422009-06-03 07:27:35 +00003507 while (!list_empty(&i915_file_priv->mm.request_list)) {
3508 struct drm_i915_gem_request *request;
3509
3510 request = list_first_entry(&i915_file_priv->mm.request_list,
3511 struct drm_i915_gem_request,
3512 client_list);
3513
3514 if (time_after_eq(request->emitted_jiffies, recent_enough))
3515 break;
3516
Zou Nan hai852835f2010-05-21 09:08:56 +08003517 ret = i915_wait_request(dev, request->seqno, request->ring);
Eric Anholtb9624422009-06-03 07:27:35 +00003518 if (ret != 0)
3519 break;
3520 }
Eric Anholt673a3942008-07-30 12:06:12 -07003521 mutex_unlock(&dev->struct_mutex);
Eric Anholtb9624422009-06-03 07:27:35 +00003522
Eric Anholt673a3942008-07-30 12:06:12 -07003523 return ret;
3524}
3525
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003526static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003527i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003528 uint32_t buffer_count,
3529 struct drm_i915_gem_relocation_entry **relocs)
3530{
3531 uint32_t reloc_count = 0, reloc_index = 0, i;
3532 int ret;
3533
3534 *relocs = NULL;
3535 for (i = 0; i < buffer_count; i++) {
3536 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3537 return -EINVAL;
3538 reloc_count += exec_list[i].relocation_count;
3539 }
3540
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003541 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
Jesse Barnes76446ca2009-12-17 22:05:42 -05003542 if (*relocs == NULL) {
3543 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003544 return -ENOMEM;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003545 }
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003546
3547 for (i = 0; i < buffer_count; i++) {
3548 struct drm_i915_gem_relocation_entry __user *user_relocs;
3549
3550 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3551
3552 ret = copy_from_user(&(*relocs)[reloc_index],
3553 user_relocs,
3554 exec_list[i].relocation_count *
3555 sizeof(**relocs));
3556 if (ret != 0) {
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003557 drm_free_large(*relocs);
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003558 *relocs = NULL;
Florian Mickler2bc43b52009-04-06 22:55:41 +02003559 return -EFAULT;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003560 }
3561
3562 reloc_index += exec_list[i].relocation_count;
3563 }
3564
Florian Mickler2bc43b52009-04-06 22:55:41 +02003565 return 0;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003566}
3567
3568static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003569i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003570 uint32_t buffer_count,
3571 struct drm_i915_gem_relocation_entry *relocs)
3572{
3573 uint32_t reloc_count = 0, i;
Florian Mickler2bc43b52009-04-06 22:55:41 +02003574 int ret = 0;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003575
Chris Wilson93533c22010-01-31 10:40:48 +00003576 if (relocs == NULL)
3577 return 0;
3578
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003579 for (i = 0; i < buffer_count; i++) {
3580 struct drm_i915_gem_relocation_entry __user *user_relocs;
Florian Mickler2bc43b52009-04-06 22:55:41 +02003581 int unwritten;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003582
3583 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3584
Florian Mickler2bc43b52009-04-06 22:55:41 +02003585 unwritten = copy_to_user(user_relocs,
3586 &relocs[reloc_count],
3587 exec_list[i].relocation_count *
3588 sizeof(*relocs));
3589
3590 if (unwritten) {
3591 ret = -EFAULT;
3592 goto err;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003593 }
3594
3595 reloc_count += exec_list[i].relocation_count;
3596 }
3597
Florian Mickler2bc43b52009-04-06 22:55:41 +02003598err:
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003599 drm_free_large(relocs);
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003600
3601 return ret;
3602}
3603
Chris Wilson83d60792009-06-06 09:45:57 +01003604static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003605i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
Chris Wilson83d60792009-06-06 09:45:57 +01003606 uint64_t exec_offset)
3607{
3608 uint32_t exec_start, exec_len;
3609
3610 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3611 exec_len = (uint32_t) exec->batch_len;
3612
3613 if ((exec_start | exec_len) & 0x7)
3614 return -EINVAL;
3615
3616 if (!exec_start)
3617 return -EINVAL;
3618
3619 return 0;
3620}
3621
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003622static int
3623i915_gem_wait_for_pending_flip(struct drm_device *dev,
3624 struct drm_gem_object **object_list,
3625 int count)
3626{
3627 drm_i915_private_t *dev_priv = dev->dev_private;
3628 struct drm_i915_gem_object *obj_priv;
3629 DEFINE_WAIT(wait);
3630 int i, ret = 0;
3631
3632 for (;;) {
3633 prepare_to_wait(&dev_priv->pending_flip_queue,
3634 &wait, TASK_INTERRUPTIBLE);
3635 for (i = 0; i < count; i++) {
Daniel Vetter23010e42010-03-08 13:35:02 +01003636 obj_priv = to_intel_bo(object_list[i]);
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003637 if (atomic_read(&obj_priv->pending_flip) > 0)
3638 break;
3639 }
3640 if (i == count)
3641 break;
3642
3643 if (!signal_pending(current)) {
3644 mutex_unlock(&dev->struct_mutex);
3645 schedule();
3646 mutex_lock(&dev->struct_mutex);
3647 continue;
3648 }
3649 ret = -ERESTARTSYS;
3650 break;
3651 }
3652 finish_wait(&dev_priv->pending_flip_queue, &wait);
3653
3654 return ret;
3655}
3656
Eric Anholt673a3942008-07-30 12:06:12 -07003657int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003658i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3659 struct drm_file *file_priv,
3660 struct drm_i915_gem_execbuffer2 *args,
3661 struct drm_i915_gem_exec_object2 *exec_list)
Eric Anholt673a3942008-07-30 12:06:12 -07003662{
3663 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07003664 struct drm_gem_object **object_list = NULL;
3665 struct drm_gem_object *batch_obj;
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003666 struct drm_i915_gem_object *obj_priv;
Eric Anholt201361a2009-03-11 12:30:04 -07003667 struct drm_clip_rect *cliprects = NULL;
Chris Wilson93533c22010-01-31 10:40:48 +00003668 struct drm_i915_gem_relocation_entry *relocs = NULL;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003669 int ret = 0, ret2, i, pinned = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07003670 uint64_t exec_offset;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003671 uint32_t seqno, flush_domains, reloc_index;
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003672 int pin_tries, flips;
Eric Anholt673a3942008-07-30 12:06:12 -07003673
Zou Nan hai852835f2010-05-21 09:08:56 +08003674 struct intel_ring_buffer *ring = NULL;
3675
Eric Anholt673a3942008-07-30 12:06:12 -07003676#if WATCH_EXEC
3677 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3678 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3679#endif
Zou Nan haid1b851f2010-05-21 09:08:57 +08003680 if (args->flags & I915_EXEC_BSD) {
3681 if (!HAS_BSD(dev)) {
3682 DRM_ERROR("execbuf with wrong flag\n");
3683 return -EINVAL;
3684 }
3685 ring = &dev_priv->bsd_ring;
3686 } else {
3687 ring = &dev_priv->render_ring;
3688 }
3689
Eric Anholt673a3942008-07-30 12:06:12 -07003690
Eric Anholt4f481ed2008-09-10 14:22:49 -07003691 if (args->buffer_count < 1) {
3692 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3693 return -EINVAL;
3694 }
Eric Anholtc8e0f932009-11-22 03:49:37 +01003695 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
Jesse Barnes76446ca2009-12-17 22:05:42 -05003696 if (object_list == NULL) {
3697 DRM_ERROR("Failed to allocate object list for %d buffers\n",
Eric Anholt673a3942008-07-30 12:06:12 -07003698 args->buffer_count);
3699 ret = -ENOMEM;
3700 goto pre_mutex_err;
3701 }
Eric Anholt673a3942008-07-30 12:06:12 -07003702
Eric Anholt201361a2009-03-11 12:30:04 -07003703 if (args->num_cliprects != 0) {
Eric Anholt9a298b22009-03-24 12:23:04 -07003704 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3705 GFP_KERNEL);
Owain Ainswortha40e8d32010-02-09 14:25:55 +00003706 if (cliprects == NULL) {
3707 ret = -ENOMEM;
Eric Anholt201361a2009-03-11 12:30:04 -07003708 goto pre_mutex_err;
Owain Ainswortha40e8d32010-02-09 14:25:55 +00003709 }
Eric Anholt201361a2009-03-11 12:30:04 -07003710
3711 ret = copy_from_user(cliprects,
3712 (struct drm_clip_rect __user *)
3713 (uintptr_t) args->cliprects_ptr,
3714 sizeof(*cliprects) * args->num_cliprects);
3715 if (ret != 0) {
3716 DRM_ERROR("copy %d cliprects failed: %d\n",
3717 args->num_cliprects, ret);
3718 goto pre_mutex_err;
3719 }
3720 }
3721
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003722 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3723 &relocs);
3724 if (ret != 0)
3725 goto pre_mutex_err;
3726
Eric Anholt673a3942008-07-30 12:06:12 -07003727 mutex_lock(&dev->struct_mutex);
3728
3729 i915_verify_inactive(dev, __FILE__, __LINE__);
3730
Ben Gamariba1234d2009-09-14 17:48:47 -04003731 if (atomic_read(&dev_priv->mm.wedged)) {
Eric Anholt673a3942008-07-30 12:06:12 -07003732 mutex_unlock(&dev->struct_mutex);
Chris Wilsona198bc82009-02-06 16:55:20 +00003733 ret = -EIO;
3734 goto pre_mutex_err;
Eric Anholt673a3942008-07-30 12:06:12 -07003735 }
3736
3737 if (dev_priv->mm.suspended) {
Eric Anholt673a3942008-07-30 12:06:12 -07003738 mutex_unlock(&dev->struct_mutex);
Chris Wilsona198bc82009-02-06 16:55:20 +00003739 ret = -EBUSY;
3740 goto pre_mutex_err;
Eric Anholt673a3942008-07-30 12:06:12 -07003741 }
3742
Keith Packardac94a962008-11-20 23:30:27 -08003743 /* Look up object handles */
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003744 flips = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07003745 for (i = 0; i < args->buffer_count; i++) {
3746 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3747 exec_list[i].handle);
3748 if (object_list[i] == NULL) {
3749 DRM_ERROR("Invalid object handle %d at index %d\n",
3750 exec_list[i].handle, i);
Chris Wilson0ce907f2010-01-23 20:26:35 +00003751 /* prevent error path from reading uninitialized data */
3752 args->buffer_count = i + 1;
Eric Anholt673a3942008-07-30 12:06:12 -07003753 ret = -EBADF;
3754 goto err;
3755 }
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003756
Daniel Vetter23010e42010-03-08 13:35:02 +01003757 obj_priv = to_intel_bo(object_list[i]);
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003758 if (obj_priv->in_execbuffer) {
3759 DRM_ERROR("Object %p appears more than once in object list\n",
3760 object_list[i]);
Chris Wilson0ce907f2010-01-23 20:26:35 +00003761 /* prevent error path from reading uninitialized data */
3762 args->buffer_count = i + 1;
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003763 ret = -EBADF;
3764 goto err;
3765 }
3766 obj_priv->in_execbuffer = true;
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003767 flips += atomic_read(&obj_priv->pending_flip);
3768 }
3769
3770 if (flips > 0) {
3771 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3772 args->buffer_count);
3773 if (ret)
3774 goto err;
Keith Packardac94a962008-11-20 23:30:27 -08003775 }
Eric Anholt673a3942008-07-30 12:06:12 -07003776
Keith Packardac94a962008-11-20 23:30:27 -08003777 /* Pin and relocate */
3778 for (pin_tries = 0; ; pin_tries++) {
3779 ret = 0;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003780 reloc_index = 0;
3781
Keith Packardac94a962008-11-20 23:30:27 -08003782 for (i = 0; i < args->buffer_count; i++) {
3783 object_list[i]->pending_read_domains = 0;
3784 object_list[i]->pending_write_domain = 0;
3785 ret = i915_gem_object_pin_and_relocate(object_list[i],
3786 file_priv,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003787 &exec_list[i],
3788 &relocs[reloc_index]);
Keith Packardac94a962008-11-20 23:30:27 -08003789 if (ret)
3790 break;
3791 pinned = i + 1;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003792 reloc_index += exec_list[i].relocation_count;
Keith Packardac94a962008-11-20 23:30:27 -08003793 }
3794 /* success */
3795 if (ret == 0)
3796 break;
3797
3798 /* error other than GTT full, or we've already tried again */
Chris Wilson2939e1f2009-06-06 09:46:03 +01003799 if (ret != -ENOSPC || pin_tries >= 1) {
Chris Wilson07f73f62009-09-14 16:50:30 +01003800 if (ret != -ERESTARTSYS) {
3801 unsigned long long total_size = 0;
Chris Wilson3d1cc472010-05-27 13:18:19 +01003802 int num_fences = 0;
3803 for (i = 0; i < args->buffer_count; i++) {
3804 obj_priv = object_list[i]->driver_private;
3805
Chris Wilson07f73f62009-09-14 16:50:30 +01003806 total_size += object_list[i]->size;
Chris Wilson3d1cc472010-05-27 13:18:19 +01003807 num_fences +=
3808 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3809 obj_priv->tiling_mode != I915_TILING_NONE;
3810 }
3811 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
Chris Wilson07f73f62009-09-14 16:50:30 +01003812 pinned+1, args->buffer_count,
Chris Wilson3d1cc472010-05-27 13:18:19 +01003813 total_size, num_fences,
3814 ret);
Chris Wilson07f73f62009-09-14 16:50:30 +01003815 DRM_ERROR("%d objects [%d pinned], "
3816 "%d object bytes [%d pinned], "
3817 "%d/%d gtt bytes\n",
3818 atomic_read(&dev->object_count),
3819 atomic_read(&dev->pin_count),
3820 atomic_read(&dev->object_memory),
3821 atomic_read(&dev->pin_memory),
3822 atomic_read(&dev->gtt_memory),
3823 dev->gtt_total);
3824 }
Eric Anholt673a3942008-07-30 12:06:12 -07003825 goto err;
3826 }
Keith Packardac94a962008-11-20 23:30:27 -08003827
3828 /* unpin all of our buffers */
3829 for (i = 0; i < pinned; i++)
3830 i915_gem_object_unpin(object_list[i]);
Eric Anholtb1177632008-12-10 10:09:41 -08003831 pinned = 0;
Keith Packardac94a962008-11-20 23:30:27 -08003832
3833 /* evict everyone we can from the aperture */
3834 ret = i915_gem_evict_everything(dev);
Chris Wilson07f73f62009-09-14 16:50:30 +01003835 if (ret && ret != -ENOSPC)
Keith Packardac94a962008-11-20 23:30:27 -08003836 goto err;
Eric Anholt673a3942008-07-30 12:06:12 -07003837 }
3838
3839 /* Set the pending read domains for the batch buffer to COMMAND */
3840 batch_obj = object_list[args->buffer_count-1];
Chris Wilson5f26a2c2009-06-06 09:45:58 +01003841 if (batch_obj->pending_write_domain) {
3842 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3843 ret = -EINVAL;
3844 goto err;
3845 }
3846 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
Eric Anholt673a3942008-07-30 12:06:12 -07003847
Chris Wilson83d60792009-06-06 09:45:57 +01003848 /* Sanity check the batch buffer, prior to moving objects */
3849 exec_offset = exec_list[args->buffer_count - 1].offset;
3850 ret = i915_gem_check_execbuffer (args, exec_offset);
3851 if (ret != 0) {
3852 DRM_ERROR("execbuf with invalid offset/length\n");
3853 goto err;
3854 }
3855
Eric Anholt673a3942008-07-30 12:06:12 -07003856 i915_verify_inactive(dev, __FILE__, __LINE__);
3857
Keith Packard646f0f62008-11-20 23:23:03 -08003858 /* Zero the global flush/invalidate flags. These
3859 * will be modified as new domains are computed
3860 * for each object
3861 */
3862 dev->invalidate_domains = 0;
3863 dev->flush_domains = 0;
3864
Eric Anholt673a3942008-07-30 12:06:12 -07003865 for (i = 0; i < args->buffer_count; i++) {
3866 struct drm_gem_object *obj = object_list[i];
Eric Anholt673a3942008-07-30 12:06:12 -07003867
Keith Packard646f0f62008-11-20 23:23:03 -08003868 /* Compute new gpu domains and update invalidate/flush */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003869 i915_gem_object_set_to_gpu_domain(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003870 }
3871
3872 i915_verify_inactive(dev, __FILE__, __LINE__);
3873
Keith Packard646f0f62008-11-20 23:23:03 -08003874 if (dev->invalidate_domains | dev->flush_domains) {
3875#if WATCH_EXEC
3876 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3877 __func__,
3878 dev->invalidate_domains,
3879 dev->flush_domains);
3880#endif
3881 i915_gem_flush(dev,
3882 dev->invalidate_domains,
3883 dev->flush_domains);
Zou Nan hai852835f2010-05-21 09:08:56 +08003884 if (dev->flush_domains & I915_GEM_GPU_DOMAINS) {
Eric Anholtb9624422009-06-03 07:27:35 +00003885 (void)i915_add_request(dev, file_priv,
Zou Nan hai852835f2010-05-21 09:08:56 +08003886 dev->flush_domains,
3887 &dev_priv->render_ring);
3888
Zou Nan haid1b851f2010-05-21 09:08:57 +08003889 if (HAS_BSD(dev))
3890 (void)i915_add_request(dev, file_priv,
3891 dev->flush_domains,
3892 &dev_priv->bsd_ring);
Zou Nan hai852835f2010-05-21 09:08:56 +08003893 }
Keith Packard646f0f62008-11-20 23:23:03 -08003894 }
Eric Anholt673a3942008-07-30 12:06:12 -07003895
Eric Anholtefbeed92009-02-19 14:54:51 -08003896 for (i = 0; i < args->buffer_count; i++) {
3897 struct drm_gem_object *obj = object_list[i];
Daniel Vetter23010e42010-03-08 13:35:02 +01003898 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003899 uint32_t old_write_domain = obj->write_domain;
Eric Anholtefbeed92009-02-19 14:54:51 -08003900
3901 obj->write_domain = obj->pending_write_domain;
Daniel Vetter99fcb762010-02-07 16:20:18 +01003902 if (obj->write_domain)
3903 list_move_tail(&obj_priv->gpu_write_list,
3904 &dev_priv->mm.gpu_write_list);
3905 else
3906 list_del_init(&obj_priv->gpu_write_list);
3907
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003908 trace_i915_gem_object_change_domain(obj,
3909 obj->read_domains,
3910 old_write_domain);
Eric Anholtefbeed92009-02-19 14:54:51 -08003911 }
3912
Eric Anholt673a3942008-07-30 12:06:12 -07003913 i915_verify_inactive(dev, __FILE__, __LINE__);
3914
3915#if WATCH_COHERENCY
3916 for (i = 0; i < args->buffer_count; i++) {
3917 i915_gem_object_check_coherency(object_list[i],
3918 exec_list[i].handle);
3919 }
3920#endif
3921
Eric Anholt673a3942008-07-30 12:06:12 -07003922#if WATCH_EXEC
Ben Gamari6911a9b2009-04-02 11:24:54 -07003923 i915_gem_dump_object(batch_obj,
Eric Anholt673a3942008-07-30 12:06:12 -07003924 args->batch_len,
3925 __func__,
3926 ~0);
3927#endif
3928
Eric Anholt673a3942008-07-30 12:06:12 -07003929 /* Exec the batchbuffer */
Zou Nan hai852835f2010-05-21 09:08:56 +08003930 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3931 cliprects, exec_offset);
Eric Anholt673a3942008-07-30 12:06:12 -07003932 if (ret) {
3933 DRM_ERROR("dispatch failed %d\n", ret);
3934 goto err;
3935 }
3936
3937 /*
3938 * Ensure that the commands in the batch buffer are
3939 * finished before the interrupt fires
3940 */
Zou Nan hai852835f2010-05-21 09:08:56 +08003941 flush_domains = i915_retire_commands(dev, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07003942
3943 i915_verify_inactive(dev, __FILE__, __LINE__);
3944
3945 /*
3946 * Get a seqno representing the execution of the current buffer,
3947 * which we can wait on. We would like to mitigate these interrupts,
3948 * likely by only creating seqnos occasionally (so that we have
3949 * *some* interrupts representing completion of buffers that we can
3950 * wait on when trying to clear up gtt space).
3951 */
Zou Nan hai852835f2010-05-21 09:08:56 +08003952 seqno = i915_add_request(dev, file_priv, flush_domains, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07003953 BUG_ON(seqno == 0);
Eric Anholt673a3942008-07-30 12:06:12 -07003954 for (i = 0; i < args->buffer_count; i++) {
3955 struct drm_gem_object *obj = object_list[i];
Zou Nan hai852835f2010-05-21 09:08:56 +08003956 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003957
Zou Nan hai852835f2010-05-21 09:08:56 +08003958 i915_gem_object_move_to_active(obj, seqno, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07003959#if WATCH_LRU
3960 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3961#endif
3962 }
3963#if WATCH_LRU
3964 i915_dump_lru(dev, __func__);
3965#endif
3966
3967 i915_verify_inactive(dev, __FILE__, __LINE__);
3968
Eric Anholt673a3942008-07-30 12:06:12 -07003969err:
Julia Lawallaad87df2008-12-21 16:28:47 +01003970 for (i = 0; i < pinned; i++)
3971 i915_gem_object_unpin(object_list[i]);
Eric Anholt673a3942008-07-30 12:06:12 -07003972
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003973 for (i = 0; i < args->buffer_count; i++) {
3974 if (object_list[i]) {
Daniel Vetter23010e42010-03-08 13:35:02 +01003975 obj_priv = to_intel_bo(object_list[i]);
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003976 obj_priv->in_execbuffer = false;
3977 }
Julia Lawallaad87df2008-12-21 16:28:47 +01003978 drm_gem_object_unreference(object_list[i]);
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003979 }
Julia Lawallaad87df2008-12-21 16:28:47 +01003980
Eric Anholt673a3942008-07-30 12:06:12 -07003981 mutex_unlock(&dev->struct_mutex);
3982
Chris Wilson93533c22010-01-31 10:40:48 +00003983pre_mutex_err:
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003984 /* Copy the updated relocations out regardless of current error
3985 * state. Failure to update the relocs would mean that the next
3986 * time userland calls execbuf, it would do so with presumed offset
3987 * state that didn't match the actual object state.
3988 */
3989 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3990 relocs);
3991 if (ret2 != 0) {
3992 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3993
3994 if (ret == 0)
3995 ret = ret2;
3996 }
3997
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003998 drm_free_large(object_list);
Eric Anholt9a298b22009-03-24 12:23:04 -07003999 kfree(cliprects);
Eric Anholt673a3942008-07-30 12:06:12 -07004000
4001 return ret;
4002}
4003
Jesse Barnes76446ca2009-12-17 22:05:42 -05004004/*
4005 * Legacy execbuffer just creates an exec2 list from the original exec object
4006 * list array and passes it to the real function.
4007 */
4008int
4009i915_gem_execbuffer(struct drm_device *dev, void *data,
4010 struct drm_file *file_priv)
4011{
4012 struct drm_i915_gem_execbuffer *args = data;
4013 struct drm_i915_gem_execbuffer2 exec2;
4014 struct drm_i915_gem_exec_object *exec_list = NULL;
4015 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4016 int ret, i;
4017
4018#if WATCH_EXEC
4019 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4020 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4021#endif
4022
4023 if (args->buffer_count < 1) {
4024 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
4025 return -EINVAL;
4026 }
4027
4028 /* Copy in the exec list from userland */
4029 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
4030 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4031 if (exec_list == NULL || exec2_list == NULL) {
4032 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4033 args->buffer_count);
4034 drm_free_large(exec_list);
4035 drm_free_large(exec2_list);
4036 return -ENOMEM;
4037 }
4038 ret = copy_from_user(exec_list,
4039 (struct drm_i915_relocation_entry __user *)
4040 (uintptr_t) args->buffers_ptr,
4041 sizeof(*exec_list) * args->buffer_count);
4042 if (ret != 0) {
4043 DRM_ERROR("copy %d exec entries failed %d\n",
4044 args->buffer_count, ret);
4045 drm_free_large(exec_list);
4046 drm_free_large(exec2_list);
4047 return -EFAULT;
4048 }
4049
4050 for (i = 0; i < args->buffer_count; i++) {
4051 exec2_list[i].handle = exec_list[i].handle;
4052 exec2_list[i].relocation_count = exec_list[i].relocation_count;
4053 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
4054 exec2_list[i].alignment = exec_list[i].alignment;
4055 exec2_list[i].offset = exec_list[i].offset;
4056 if (!IS_I965G(dev))
4057 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
4058 else
4059 exec2_list[i].flags = 0;
4060 }
4061
4062 exec2.buffers_ptr = args->buffers_ptr;
4063 exec2.buffer_count = args->buffer_count;
4064 exec2.batch_start_offset = args->batch_start_offset;
4065 exec2.batch_len = args->batch_len;
4066 exec2.DR1 = args->DR1;
4067 exec2.DR4 = args->DR4;
4068 exec2.num_cliprects = args->num_cliprects;
4069 exec2.cliprects_ptr = args->cliprects_ptr;
Zou Nan hai852835f2010-05-21 09:08:56 +08004070 exec2.flags = I915_EXEC_RENDER;
Jesse Barnes76446ca2009-12-17 22:05:42 -05004071
4072 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
4073 if (!ret) {
4074 /* Copy the new buffer offsets back to the user's exec list. */
4075 for (i = 0; i < args->buffer_count; i++)
4076 exec_list[i].offset = exec2_list[i].offset;
4077 /* ... and back out to userspace */
4078 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4079 (uintptr_t) args->buffers_ptr,
4080 exec_list,
4081 sizeof(*exec_list) * args->buffer_count);
4082 if (ret) {
4083 ret = -EFAULT;
4084 DRM_ERROR("failed to copy %d exec entries "
4085 "back to user (%d)\n",
4086 args->buffer_count, ret);
4087 }
Jesse Barnes76446ca2009-12-17 22:05:42 -05004088 }
4089
4090 drm_free_large(exec_list);
4091 drm_free_large(exec2_list);
4092 return ret;
4093}
4094
4095int
4096i915_gem_execbuffer2(struct drm_device *dev, void *data,
4097 struct drm_file *file_priv)
4098{
4099 struct drm_i915_gem_execbuffer2 *args = data;
4100 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4101 int ret;
4102
4103#if WATCH_EXEC
4104 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4105 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4106#endif
4107
4108 if (args->buffer_count < 1) {
4109 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4110 return -EINVAL;
4111 }
4112
4113 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4114 if (exec2_list == NULL) {
4115 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4116 args->buffer_count);
4117 return -ENOMEM;
4118 }
4119 ret = copy_from_user(exec2_list,
4120 (struct drm_i915_relocation_entry __user *)
4121 (uintptr_t) args->buffers_ptr,
4122 sizeof(*exec2_list) * args->buffer_count);
4123 if (ret != 0) {
4124 DRM_ERROR("copy %d exec entries failed %d\n",
4125 args->buffer_count, ret);
4126 drm_free_large(exec2_list);
4127 return -EFAULT;
4128 }
4129
4130 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4131 if (!ret) {
4132 /* Copy the new buffer offsets back to the user's exec list. */
4133 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4134 (uintptr_t) args->buffers_ptr,
4135 exec2_list,
4136 sizeof(*exec2_list) * args->buffer_count);
4137 if (ret) {
4138 ret = -EFAULT;
4139 DRM_ERROR("failed to copy %d exec entries "
4140 "back to user (%d)\n",
4141 args->buffer_count, ret);
4142 }
4143 }
4144
4145 drm_free_large(exec2_list);
4146 return ret;
4147}
4148
Eric Anholt673a3942008-07-30 12:06:12 -07004149int
4150i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4151{
4152 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01004153 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004154 int ret;
4155
Daniel Vetter778c3542010-05-13 11:49:44 +02004156 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4157
Eric Anholt673a3942008-07-30 12:06:12 -07004158 i915_verify_inactive(dev, __FILE__, __LINE__);
Chris Wilsonac0c6b52010-05-27 13:18:18 +01004159
4160 if (obj_priv->gtt_space != NULL) {
4161 if (alignment == 0)
4162 alignment = i915_gem_get_gtt_alignment(obj);
4163 if (obj_priv->gtt_offset & (alignment - 1)) {
4164 ret = i915_gem_object_unbind(obj);
4165 if (ret)
4166 return ret;
4167 }
4168 }
4169
Eric Anholt673a3942008-07-30 12:06:12 -07004170 if (obj_priv->gtt_space == NULL) {
4171 ret = i915_gem_object_bind_to_gtt(obj, alignment);
Chris Wilson97311292009-09-21 00:22:34 +01004172 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07004173 return ret;
Chris Wilson22c344e2009-02-11 14:26:45 +00004174 }
Jesse Barnes76446ca2009-12-17 22:05:42 -05004175
Eric Anholt673a3942008-07-30 12:06:12 -07004176 obj_priv->pin_count++;
4177
4178 /* If the object is not active and not pending a flush,
4179 * remove it from the inactive list
4180 */
4181 if (obj_priv->pin_count == 1) {
4182 atomic_inc(&dev->pin_count);
4183 atomic_add(obj->size, &dev->pin_memory);
4184 if (!obj_priv->active &&
Chris Wilson21d509e2009-06-06 09:46:02 +01004185 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0 &&
Eric Anholt673a3942008-07-30 12:06:12 -07004186 !list_empty(&obj_priv->list))
4187 list_del_init(&obj_priv->list);
4188 }
4189 i915_verify_inactive(dev, __FILE__, __LINE__);
4190
4191 return 0;
4192}
4193
4194void
4195i915_gem_object_unpin(struct drm_gem_object *obj)
4196{
4197 struct drm_device *dev = obj->dev;
4198 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01004199 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004200
4201 i915_verify_inactive(dev, __FILE__, __LINE__);
4202 obj_priv->pin_count--;
4203 BUG_ON(obj_priv->pin_count < 0);
4204 BUG_ON(obj_priv->gtt_space == NULL);
4205
4206 /* If the object is no longer pinned, and is
4207 * neither active nor being flushed, then stick it on
4208 * the inactive list
4209 */
4210 if (obj_priv->pin_count == 0) {
4211 if (!obj_priv->active &&
Chris Wilson21d509e2009-06-06 09:46:02 +01004212 (obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
Eric Anholt673a3942008-07-30 12:06:12 -07004213 list_move_tail(&obj_priv->list,
4214 &dev_priv->mm.inactive_list);
4215 atomic_dec(&dev->pin_count);
4216 atomic_sub(obj->size, &dev->pin_memory);
4217 }
4218 i915_verify_inactive(dev, __FILE__, __LINE__);
4219}
4220
4221int
4222i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4223 struct drm_file *file_priv)
4224{
4225 struct drm_i915_gem_pin *args = data;
4226 struct drm_gem_object *obj;
4227 struct drm_i915_gem_object *obj_priv;
4228 int ret;
4229
4230 mutex_lock(&dev->struct_mutex);
4231
4232 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4233 if (obj == NULL) {
4234 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4235 args->handle);
4236 mutex_unlock(&dev->struct_mutex);
4237 return -EBADF;
4238 }
Daniel Vetter23010e42010-03-08 13:35:02 +01004239 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004240
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004241 if (obj_priv->madv != I915_MADV_WILLNEED) {
4242 DRM_ERROR("Attempting to pin a purgeable buffer\n");
Chris Wilson3ef94da2009-09-14 16:50:29 +01004243 drm_gem_object_unreference(obj);
4244 mutex_unlock(&dev->struct_mutex);
4245 return -EINVAL;
4246 }
4247
Jesse Barnes79e53942008-11-07 14:24:08 -08004248 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4249 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4250 args->handle);
Chris Wilson96dec612009-02-08 19:08:04 +00004251 drm_gem_object_unreference(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004252 mutex_unlock(&dev->struct_mutex);
Jesse Barnes79e53942008-11-07 14:24:08 -08004253 return -EINVAL;
4254 }
4255
4256 obj_priv->user_pin_count++;
4257 obj_priv->pin_filp = file_priv;
4258 if (obj_priv->user_pin_count == 1) {
4259 ret = i915_gem_object_pin(obj, args->alignment);
4260 if (ret != 0) {
4261 drm_gem_object_unreference(obj);
4262 mutex_unlock(&dev->struct_mutex);
4263 return ret;
4264 }
Eric Anholt673a3942008-07-30 12:06:12 -07004265 }
4266
4267 /* XXX - flush the CPU caches for pinned objects
4268 * as the X server doesn't manage domains yet
4269 */
Eric Anholte47c68e2008-11-14 13:35:19 -08004270 i915_gem_object_flush_cpu_write_domain(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004271 args->offset = obj_priv->gtt_offset;
4272 drm_gem_object_unreference(obj);
4273 mutex_unlock(&dev->struct_mutex);
4274
4275 return 0;
4276}
4277
4278int
4279i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4280 struct drm_file *file_priv)
4281{
4282 struct drm_i915_gem_pin *args = data;
4283 struct drm_gem_object *obj;
Jesse Barnes79e53942008-11-07 14:24:08 -08004284 struct drm_i915_gem_object *obj_priv;
Eric Anholt673a3942008-07-30 12:06:12 -07004285
4286 mutex_lock(&dev->struct_mutex);
4287
4288 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4289 if (obj == NULL) {
4290 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4291 args->handle);
4292 mutex_unlock(&dev->struct_mutex);
4293 return -EBADF;
4294 }
4295
Daniel Vetter23010e42010-03-08 13:35:02 +01004296 obj_priv = to_intel_bo(obj);
Jesse Barnes79e53942008-11-07 14:24:08 -08004297 if (obj_priv->pin_filp != file_priv) {
4298 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4299 args->handle);
4300 drm_gem_object_unreference(obj);
4301 mutex_unlock(&dev->struct_mutex);
4302 return -EINVAL;
4303 }
4304 obj_priv->user_pin_count--;
4305 if (obj_priv->user_pin_count == 0) {
4306 obj_priv->pin_filp = NULL;
4307 i915_gem_object_unpin(obj);
4308 }
Eric Anholt673a3942008-07-30 12:06:12 -07004309
4310 drm_gem_object_unreference(obj);
4311 mutex_unlock(&dev->struct_mutex);
4312 return 0;
4313}
4314
4315int
4316i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4317 struct drm_file *file_priv)
4318{
4319 struct drm_i915_gem_busy *args = data;
4320 struct drm_gem_object *obj;
4321 struct drm_i915_gem_object *obj_priv;
Zou Nan hai852835f2010-05-21 09:08:56 +08004322 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07004323
Eric Anholt673a3942008-07-30 12:06:12 -07004324 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4325 if (obj == NULL) {
4326 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4327 args->handle);
Eric Anholt673a3942008-07-30 12:06:12 -07004328 return -EBADF;
4329 }
4330
Chris Wilsonb1ce7862009-06-06 09:46:00 +01004331 mutex_lock(&dev->struct_mutex);
Eric Anholtf21289b2009-02-18 09:44:56 -08004332 /* Update the active list for the hardware's current position.
4333 * Otherwise this only updates on a delayed timer or when irqs are
4334 * actually unmasked, and our working set ends up being larger than
4335 * required.
4336 */
Zou Nan hai852835f2010-05-21 09:08:56 +08004337 i915_gem_retire_requests(dev, &dev_priv->render_ring);
Eric Anholtf21289b2009-02-18 09:44:56 -08004338
Zou Nan haid1b851f2010-05-21 09:08:57 +08004339 if (HAS_BSD(dev))
4340 i915_gem_retire_requests(dev, &dev_priv->bsd_ring);
4341
Daniel Vetter23010e42010-03-08 13:35:02 +01004342 obj_priv = to_intel_bo(obj);
Eric Anholtc4de0a52008-12-14 19:05:04 -08004343 /* Don't count being on the flushing list against the object being
4344 * done. Otherwise, a buffer left on the flushing list but not getting
4345 * flushed (because nobody's flushing that domain) won't ever return
4346 * unbusy and get reused by libdrm's bo cache. The other expected
4347 * consumer of this interface, OpenGL's occlusion queries, also specs
4348 * that the objects get unbusy "eventually" without any interference.
4349 */
4350 args->busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
Eric Anholt673a3942008-07-30 12:06:12 -07004351
4352 drm_gem_object_unreference(obj);
4353 mutex_unlock(&dev->struct_mutex);
4354 return 0;
4355}
4356
4357int
4358i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4359 struct drm_file *file_priv)
4360{
4361 return i915_gem_ring_throttle(dev, file_priv);
4362}
4363
Chris Wilson3ef94da2009-09-14 16:50:29 +01004364int
4365i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4366 struct drm_file *file_priv)
4367{
4368 struct drm_i915_gem_madvise *args = data;
4369 struct drm_gem_object *obj;
4370 struct drm_i915_gem_object *obj_priv;
4371
4372 switch (args->madv) {
4373 case I915_MADV_DONTNEED:
4374 case I915_MADV_WILLNEED:
4375 break;
4376 default:
4377 return -EINVAL;
4378 }
4379
4380 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4381 if (obj == NULL) {
4382 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4383 args->handle);
4384 return -EBADF;
4385 }
4386
4387 mutex_lock(&dev->struct_mutex);
Daniel Vetter23010e42010-03-08 13:35:02 +01004388 obj_priv = to_intel_bo(obj);
Chris Wilson3ef94da2009-09-14 16:50:29 +01004389
4390 if (obj_priv->pin_count) {
4391 drm_gem_object_unreference(obj);
4392 mutex_unlock(&dev->struct_mutex);
4393
4394 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4395 return -EINVAL;
4396 }
4397
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004398 if (obj_priv->madv != __I915_MADV_PURGED)
4399 obj_priv->madv = args->madv;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004400
Chris Wilson2d7ef392009-09-20 23:13:10 +01004401 /* if the object is no longer bound, discard its backing storage */
4402 if (i915_gem_object_is_purgeable(obj_priv) &&
4403 obj_priv->gtt_space == NULL)
4404 i915_gem_object_truncate(obj);
4405
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004406 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4407
Chris Wilson3ef94da2009-09-14 16:50:29 +01004408 drm_gem_object_unreference(obj);
4409 mutex_unlock(&dev->struct_mutex);
4410
4411 return 0;
4412}
4413
Daniel Vetterac52bc52010-04-09 19:05:06 +00004414struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4415 size_t size)
4416{
Daniel Vetterc397b902010-04-09 19:05:07 +00004417 struct drm_i915_gem_object *obj;
4418
4419 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4420 if (obj == NULL)
4421 return NULL;
4422
4423 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4424 kfree(obj);
4425 return NULL;
4426 }
4427
4428 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4429 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4430
4431 obj->agp_type = AGP_USER_MEMORY;
Daniel Vetter62b8b212010-04-09 19:05:08 +00004432 obj->base.driver_private = NULL;
Daniel Vetterc397b902010-04-09 19:05:07 +00004433 obj->fence_reg = I915_FENCE_REG_NONE;
4434 INIT_LIST_HEAD(&obj->list);
4435 INIT_LIST_HEAD(&obj->gpu_write_list);
Daniel Vetterc397b902010-04-09 19:05:07 +00004436 obj->madv = I915_MADV_WILLNEED;
4437
4438 trace_i915_gem_object_create(&obj->base);
4439
4440 return &obj->base;
Daniel Vetterac52bc52010-04-09 19:05:06 +00004441}
4442
Eric Anholt673a3942008-07-30 12:06:12 -07004443int i915_gem_init_object(struct drm_gem_object *obj)
4444{
Daniel Vetterc397b902010-04-09 19:05:07 +00004445 BUG();
Jesse Barnesde151cf2008-11-12 10:03:55 -08004446
Eric Anholt673a3942008-07-30 12:06:12 -07004447 return 0;
4448}
4449
4450void i915_gem_free_object(struct drm_gem_object *obj)
4451{
Jesse Barnesde151cf2008-11-12 10:03:55 -08004452 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01004453 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004454
Chris Wilson1c5d22f2009-08-25 11:15:50 +01004455 trace_i915_gem_object_destroy(obj);
4456
Eric Anholt673a3942008-07-30 12:06:12 -07004457 while (obj_priv->pin_count > 0)
4458 i915_gem_object_unpin(obj);
4459
Dave Airlie71acb5e2008-12-30 20:31:46 +10004460 if (obj_priv->phys_obj)
4461 i915_gem_detach_phys_object(dev, obj);
4462
Eric Anholt673a3942008-07-30 12:06:12 -07004463 i915_gem_object_unbind(obj);
4464
Chris Wilson7e616152009-09-10 08:53:04 +01004465 if (obj_priv->mmap_offset)
4466 i915_gem_free_mmap_offset(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08004467
Daniel Vetterc397b902010-04-09 19:05:07 +00004468 drm_gem_object_release(obj);
4469
Eric Anholt9a298b22009-03-24 12:23:04 -07004470 kfree(obj_priv->page_cpu_valid);
Eric Anholt280b7132009-03-12 16:56:27 -07004471 kfree(obj_priv->bit_17);
Daniel Vetterc397b902010-04-09 19:05:07 +00004472 kfree(obj_priv);
Eric Anholt673a3942008-07-30 12:06:12 -07004473}
4474
Chris Wilsonab5ee572009-09-20 19:25:47 +01004475/** Unbinds all inactive objects. */
Eric Anholt673a3942008-07-30 12:06:12 -07004476static int
Chris Wilsonab5ee572009-09-20 19:25:47 +01004477i915_gem_evict_from_inactive_list(struct drm_device *dev)
Eric Anholt673a3942008-07-30 12:06:12 -07004478{
Chris Wilsonab5ee572009-09-20 19:25:47 +01004479 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07004480
Chris Wilsonab5ee572009-09-20 19:25:47 +01004481 while (!list_empty(&dev_priv->mm.inactive_list)) {
4482 struct drm_gem_object *obj;
4483 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004484
Daniel Vettera8089e82010-04-09 19:05:09 +00004485 obj = &list_first_entry(&dev_priv->mm.inactive_list,
4486 struct drm_i915_gem_object,
4487 list)->base;
Eric Anholt673a3942008-07-30 12:06:12 -07004488
4489 ret = i915_gem_object_unbind(obj);
4490 if (ret != 0) {
Chris Wilsonab5ee572009-09-20 19:25:47 +01004491 DRM_ERROR("Error unbinding object: %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07004492 return ret;
4493 }
4494 }
4495
Eric Anholt673a3942008-07-30 12:06:12 -07004496 return 0;
4497}
4498
Jesse Barnes5669fca2009-02-17 15:13:31 -08004499int
Eric Anholt673a3942008-07-30 12:06:12 -07004500i915_gem_idle(struct drm_device *dev)
4501{
4502 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson29105cc2010-01-07 10:39:13 +00004503 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004504
Keith Packard6dbe2772008-10-14 21:41:13 -07004505 mutex_lock(&dev->struct_mutex);
4506
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004507 if (dev_priv->mm.suspended ||
Zou Nan haid1b851f2010-05-21 09:08:57 +08004508 (dev_priv->render_ring.gem_object == NULL) ||
4509 (HAS_BSD(dev) &&
4510 dev_priv->bsd_ring.gem_object == NULL)) {
Keith Packard6dbe2772008-10-14 21:41:13 -07004511 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07004512 return 0;
Keith Packard6dbe2772008-10-14 21:41:13 -07004513 }
Eric Anholt673a3942008-07-30 12:06:12 -07004514
Chris Wilson29105cc2010-01-07 10:39:13 +00004515 ret = i915_gpu_idle(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07004516 if (ret) {
4517 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07004518 return ret;
Keith Packard6dbe2772008-10-14 21:41:13 -07004519 }
Eric Anholt673a3942008-07-30 12:06:12 -07004520
Chris Wilson29105cc2010-01-07 10:39:13 +00004521 /* Under UMS, be paranoid and evict. */
4522 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
4523 ret = i915_gem_evict_from_inactive_list(dev);
4524 if (ret) {
4525 mutex_unlock(&dev->struct_mutex);
4526 return ret;
4527 }
4528 }
4529
4530 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4531 * We need to replace this with a semaphore, or something.
4532 * And not confound mm.suspended!
4533 */
4534 dev_priv->mm.suspended = 1;
4535 del_timer(&dev_priv->hangcheck_timer);
4536
4537 i915_kernel_lost_context(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07004538 i915_gem_cleanup_ringbuffer(dev);
Chris Wilson29105cc2010-01-07 10:39:13 +00004539
Keith Packard6dbe2772008-10-14 21:41:13 -07004540 mutex_unlock(&dev->struct_mutex);
4541
Chris Wilson29105cc2010-01-07 10:39:13 +00004542 /* Cancel the retire work handler, which should be idle now. */
4543 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4544
Eric Anholt673a3942008-07-30 12:06:12 -07004545 return 0;
4546}
4547
Jesse Barnese552eb72010-04-21 11:39:23 -07004548/*
4549 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4550 * over cache flushing.
4551 */
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004552static int
Jesse Barnese552eb72010-04-21 11:39:23 -07004553i915_gem_init_pipe_control(struct drm_device *dev)
4554{
4555 drm_i915_private_t *dev_priv = dev->dev_private;
4556 struct drm_gem_object *obj;
4557 struct drm_i915_gem_object *obj_priv;
4558 int ret;
4559
Eric Anholt34dc4d42010-05-07 14:30:03 -07004560 obj = i915_gem_alloc_object(dev, 4096);
Jesse Barnese552eb72010-04-21 11:39:23 -07004561 if (obj == NULL) {
4562 DRM_ERROR("Failed to allocate seqno page\n");
4563 ret = -ENOMEM;
4564 goto err;
4565 }
4566 obj_priv = to_intel_bo(obj);
4567 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4568
4569 ret = i915_gem_object_pin(obj, 4096);
4570 if (ret)
4571 goto err_unref;
4572
4573 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4574 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4575 if (dev_priv->seqno_page == NULL)
4576 goto err_unpin;
4577
4578 dev_priv->seqno_obj = obj;
4579 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4580
4581 return 0;
4582
4583err_unpin:
4584 i915_gem_object_unpin(obj);
4585err_unref:
4586 drm_gem_object_unreference(obj);
4587err:
4588 return ret;
4589}
4590
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004591
4592static void
Jesse Barnese552eb72010-04-21 11:39:23 -07004593i915_gem_cleanup_pipe_control(struct drm_device *dev)
4594{
4595 drm_i915_private_t *dev_priv = dev->dev_private;
4596 struct drm_gem_object *obj;
4597 struct drm_i915_gem_object *obj_priv;
4598
4599 obj = dev_priv->seqno_obj;
4600 obj_priv = to_intel_bo(obj);
4601 kunmap(obj_priv->pages[0]);
4602 i915_gem_object_unpin(obj);
4603 drm_gem_object_unreference(obj);
4604 dev_priv->seqno_obj = NULL;
4605
4606 dev_priv->seqno_page = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07004607}
4608
Eric Anholt673a3942008-07-30 12:06:12 -07004609int
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004610i915_gem_init_ringbuffer(struct drm_device *dev)
4611{
4612 drm_i915_private_t *dev_priv = dev->dev_private;
4613 int ret;
Chris Wilson68f95ba2010-05-27 13:18:22 +01004614
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004615 dev_priv->render_ring = render_ring;
Chris Wilson68f95ba2010-05-27 13:18:22 +01004616
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004617 if (!I915_NEED_GFX_HWS(dev)) {
4618 dev_priv->render_ring.status_page.page_addr
4619 = dev_priv->status_page_dmah->vaddr;
4620 memset(dev_priv->render_ring.status_page.page_addr,
4621 0, PAGE_SIZE);
4622 }
Chris Wilson68f95ba2010-05-27 13:18:22 +01004623
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004624 if (HAS_PIPE_CONTROL(dev)) {
4625 ret = i915_gem_init_pipe_control(dev);
4626 if (ret)
4627 return ret;
4628 }
Chris Wilson68f95ba2010-05-27 13:18:22 +01004629
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004630 ret = intel_init_ring_buffer(dev, &dev_priv->render_ring);
Chris Wilson68f95ba2010-05-27 13:18:22 +01004631 if (ret)
4632 goto cleanup_pipe_control;
4633
4634 if (HAS_BSD(dev)) {
Zou Nan haid1b851f2010-05-21 09:08:57 +08004635 dev_priv->bsd_ring = bsd_ring;
4636 ret = intel_init_ring_buffer(dev, &dev_priv->bsd_ring);
Chris Wilson68f95ba2010-05-27 13:18:22 +01004637 if (ret)
4638 goto cleanup_render_ring;
Zou Nan haid1b851f2010-05-21 09:08:57 +08004639 }
Chris Wilson68f95ba2010-05-27 13:18:22 +01004640
4641 return 0;
4642
4643cleanup_render_ring:
4644 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4645cleanup_pipe_control:
4646 if (HAS_PIPE_CONTROL(dev))
4647 i915_gem_cleanup_pipe_control(dev);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004648 return ret;
4649}
4650
4651void
4652i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4653{
4654 drm_i915_private_t *dev_priv = dev->dev_private;
4655
4656 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004657 if (HAS_BSD(dev))
4658 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004659 if (HAS_PIPE_CONTROL(dev))
4660 i915_gem_cleanup_pipe_control(dev);
4661}
4662
4663int
Eric Anholt673a3942008-07-30 12:06:12 -07004664i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4665 struct drm_file *file_priv)
4666{
4667 drm_i915_private_t *dev_priv = dev->dev_private;
4668 int ret;
4669
Jesse Barnes79e53942008-11-07 14:24:08 -08004670 if (drm_core_check_feature(dev, DRIVER_MODESET))
4671 return 0;
4672
Ben Gamariba1234d2009-09-14 17:48:47 -04004673 if (atomic_read(&dev_priv->mm.wedged)) {
Eric Anholt673a3942008-07-30 12:06:12 -07004674 DRM_ERROR("Reenabling wedged hardware, good luck\n");
Ben Gamariba1234d2009-09-14 17:48:47 -04004675 atomic_set(&dev_priv->mm.wedged, 0);
Eric Anholt673a3942008-07-30 12:06:12 -07004676 }
4677
Eric Anholt673a3942008-07-30 12:06:12 -07004678 mutex_lock(&dev->struct_mutex);
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004679 dev_priv->mm.suspended = 0;
4680
4681 ret = i915_gem_init_ringbuffer(dev);
Wu Fengguangd816f6a2009-04-18 10:43:32 +08004682 if (ret != 0) {
4683 mutex_unlock(&dev->struct_mutex);
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004684 return ret;
Wu Fengguangd816f6a2009-04-18 10:43:32 +08004685 }
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004686
Carl Worth5e118f42009-03-20 11:54:25 -07004687 spin_lock(&dev_priv->mm.active_list_lock);
Zou Nan hai852835f2010-05-21 09:08:56 +08004688 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
Zou Nan haid1b851f2010-05-21 09:08:57 +08004689 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
Carl Worth5e118f42009-03-20 11:54:25 -07004690 spin_unlock(&dev_priv->mm.active_list_lock);
4691
Eric Anholt673a3942008-07-30 12:06:12 -07004692 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4693 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
Zou Nan hai852835f2010-05-21 09:08:56 +08004694 BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
Zou Nan haid1b851f2010-05-21 09:08:57 +08004695 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
Eric Anholt673a3942008-07-30 12:06:12 -07004696 mutex_unlock(&dev->struct_mutex);
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004697
4698 drm_irq_install(dev);
4699
Eric Anholt673a3942008-07-30 12:06:12 -07004700 return 0;
4701}
4702
4703int
4704i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4705 struct drm_file *file_priv)
4706{
Jesse Barnes79e53942008-11-07 14:24:08 -08004707 if (drm_core_check_feature(dev, DRIVER_MODESET))
4708 return 0;
4709
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004710 drm_irq_uninstall(dev);
Linus Torvaldse6890f62009-09-08 17:09:24 -07004711 return i915_gem_idle(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07004712}
4713
4714void
4715i915_gem_lastclose(struct drm_device *dev)
4716{
4717 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004718
Eric Anholte806b492009-01-22 09:56:58 -08004719 if (drm_core_check_feature(dev, DRIVER_MODESET))
4720 return;
4721
Keith Packard6dbe2772008-10-14 21:41:13 -07004722 ret = i915_gem_idle(dev);
4723 if (ret)
4724 DRM_ERROR("failed to idle hardware: %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07004725}
4726
4727void
4728i915_gem_load(struct drm_device *dev)
4729{
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004730 int i;
Eric Anholt673a3942008-07-30 12:06:12 -07004731 drm_i915_private_t *dev_priv = dev->dev_private;
4732
Carl Worth5e118f42009-03-20 11:54:25 -07004733 spin_lock_init(&dev_priv->mm.active_list_lock);
Eric Anholt673a3942008-07-30 12:06:12 -07004734 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
Daniel Vetter99fcb762010-02-07 16:20:18 +01004735 INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004736 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07004737 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
Zou Nan hai852835f2010-05-21 09:08:56 +08004738 INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4739 INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004740 if (HAS_BSD(dev)) {
4741 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4742 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4743 }
Daniel Vetter007cc8a2010-04-28 11:02:31 +02004744 for (i = 0; i < 16; i++)
4745 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004746 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4747 i915_gem_retire_work_handler);
Chris Wilson31169712009-09-14 16:50:28 +01004748 spin_lock(&shrink_list_lock);
4749 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4750 spin_unlock(&shrink_list_lock);
4751
Jesse Barnesde151cf2008-11-12 10:03:55 -08004752 /* Old X drivers will take 0-2 for front, back, depth buffers */
Eric Anholtb397c832010-01-26 09:43:10 -08004753 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4754 dev_priv->fence_reg_start = 3;
Jesse Barnesde151cf2008-11-12 10:03:55 -08004755
Jesse Barnes0f973f22009-01-26 17:10:45 -08004756 if (IS_I965G(dev) || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
Jesse Barnesde151cf2008-11-12 10:03:55 -08004757 dev_priv->num_fence_regs = 16;
4758 else
4759 dev_priv->num_fence_regs = 8;
4760
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004761 /* Initialize fence registers to zero */
4762 if (IS_I965G(dev)) {
4763 for (i = 0; i < 16; i++)
4764 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
4765 } else {
4766 for (i = 0; i < 8; i++)
4767 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4768 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4769 for (i = 0; i < 8; i++)
4770 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
4771 }
Eric Anholt673a3942008-07-30 12:06:12 -07004772 i915_gem_detect_bit_6_swizzle(dev);
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05004773 init_waitqueue_head(&dev_priv->pending_flip_queue);
Eric Anholt673a3942008-07-30 12:06:12 -07004774}
Dave Airlie71acb5e2008-12-30 20:31:46 +10004775
4776/*
4777 * Create a physically contiguous memory object for this object
4778 * e.g. for cursor + overlay regs
4779 */
4780int i915_gem_init_phys_object(struct drm_device *dev,
4781 int id, int size)
4782{
4783 drm_i915_private_t *dev_priv = dev->dev_private;
4784 struct drm_i915_gem_phys_object *phys_obj;
4785 int ret;
4786
4787 if (dev_priv->mm.phys_objs[id - 1] || !size)
4788 return 0;
4789
Eric Anholt9a298b22009-03-24 12:23:04 -07004790 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004791 if (!phys_obj)
4792 return -ENOMEM;
4793
4794 phys_obj->id = id;
4795
Zhenyu Wange6be8d92010-01-05 11:25:05 +08004796 phys_obj->handle = drm_pci_alloc(dev, size, 0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004797 if (!phys_obj->handle) {
4798 ret = -ENOMEM;
4799 goto kfree_obj;
4800 }
4801#ifdef CONFIG_X86
4802 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4803#endif
4804
4805 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4806
4807 return 0;
4808kfree_obj:
Eric Anholt9a298b22009-03-24 12:23:04 -07004809 kfree(phys_obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004810 return ret;
4811}
4812
4813void i915_gem_free_phys_object(struct drm_device *dev, int id)
4814{
4815 drm_i915_private_t *dev_priv = dev->dev_private;
4816 struct drm_i915_gem_phys_object *phys_obj;
4817
4818 if (!dev_priv->mm.phys_objs[id - 1])
4819 return;
4820
4821 phys_obj = dev_priv->mm.phys_objs[id - 1];
4822 if (phys_obj->cur_obj) {
4823 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4824 }
4825
4826#ifdef CONFIG_X86
4827 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4828#endif
4829 drm_pci_free(dev, phys_obj->handle);
4830 kfree(phys_obj);
4831 dev_priv->mm.phys_objs[id - 1] = NULL;
4832}
4833
4834void i915_gem_free_all_phys_object(struct drm_device *dev)
4835{
4836 int i;
4837
Dave Airlie260883c2009-01-22 17:58:49 +10004838 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004839 i915_gem_free_phys_object(dev, i);
4840}
4841
4842void i915_gem_detach_phys_object(struct drm_device *dev,
4843 struct drm_gem_object *obj)
4844{
4845 struct drm_i915_gem_object *obj_priv;
4846 int i;
4847 int ret;
4848 int page_count;
4849
Daniel Vetter23010e42010-03-08 13:35:02 +01004850 obj_priv = to_intel_bo(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004851 if (!obj_priv->phys_obj)
4852 return;
4853
Chris Wilson4bdadb92010-01-27 13:36:32 +00004854 ret = i915_gem_object_get_pages(obj, 0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004855 if (ret)
4856 goto out;
4857
4858 page_count = obj->size / PAGE_SIZE;
4859
4860 for (i = 0; i < page_count; i++) {
Eric Anholt856fa192009-03-19 14:10:50 -07004861 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004862 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4863
4864 memcpy(dst, src, PAGE_SIZE);
4865 kunmap_atomic(dst, KM_USER0);
4866 }
Eric Anholt856fa192009-03-19 14:10:50 -07004867 drm_clflush_pages(obj_priv->pages, page_count);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004868 drm_agp_chipset_flush(dev);
Chris Wilsond78b47b2009-06-17 21:52:49 +01004869
4870 i915_gem_object_put_pages(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004871out:
4872 obj_priv->phys_obj->cur_obj = NULL;
4873 obj_priv->phys_obj = NULL;
4874}
4875
4876int
4877i915_gem_attach_phys_object(struct drm_device *dev,
4878 struct drm_gem_object *obj, int id)
4879{
4880 drm_i915_private_t *dev_priv = dev->dev_private;
4881 struct drm_i915_gem_object *obj_priv;
4882 int ret = 0;
4883 int page_count;
4884 int i;
4885
4886 if (id > I915_MAX_PHYS_OBJECT)
4887 return -EINVAL;
4888
Daniel Vetter23010e42010-03-08 13:35:02 +01004889 obj_priv = to_intel_bo(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004890
4891 if (obj_priv->phys_obj) {
4892 if (obj_priv->phys_obj->id == id)
4893 return 0;
4894 i915_gem_detach_phys_object(dev, obj);
4895 }
4896
4897
4898 /* create a new object */
4899 if (!dev_priv->mm.phys_objs[id - 1]) {
4900 ret = i915_gem_init_phys_object(dev, id,
4901 obj->size);
4902 if (ret) {
Linus Torvaldsaeb565d2009-01-26 10:01:53 -08004903 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004904 goto out;
4905 }
4906 }
4907
4908 /* bind to the object */
4909 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4910 obj_priv->phys_obj->cur_obj = obj;
4911
Chris Wilson4bdadb92010-01-27 13:36:32 +00004912 ret = i915_gem_object_get_pages(obj, 0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004913 if (ret) {
4914 DRM_ERROR("failed to get page list\n");
4915 goto out;
4916 }
4917
4918 page_count = obj->size / PAGE_SIZE;
4919
4920 for (i = 0; i < page_count; i++) {
Eric Anholt856fa192009-03-19 14:10:50 -07004921 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004922 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4923
4924 memcpy(dst, src, PAGE_SIZE);
4925 kunmap_atomic(src, KM_USER0);
4926 }
4927
Chris Wilsond78b47b2009-06-17 21:52:49 +01004928 i915_gem_object_put_pages(obj);
4929
Dave Airlie71acb5e2008-12-30 20:31:46 +10004930 return 0;
4931out:
4932 return ret;
4933}
4934
4935static int
4936i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4937 struct drm_i915_gem_pwrite *args,
4938 struct drm_file *file_priv)
4939{
Daniel Vetter23010e42010-03-08 13:35:02 +01004940 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004941 void *obj_addr;
4942 int ret;
4943 char __user *user_data;
4944
4945 user_data = (char __user *) (uintptr_t) args->data_ptr;
4946 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4947
Zhao Yakui44d98a62009-10-09 11:39:40 +08004948 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004949 ret = copy_from_user(obj_addr, user_data, args->size);
4950 if (ret)
4951 return -EFAULT;
4952
4953 drm_agp_chipset_flush(dev);
4954 return 0;
4955}
Eric Anholtb9624422009-06-03 07:27:35 +00004956
4957void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4958{
4959 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4960
4961 /* Clean up our request list when the client is going away, so that
4962 * later retire_requests won't dereference our soon-to-be-gone
4963 * file_priv.
4964 */
4965 mutex_lock(&dev->struct_mutex);
4966 while (!list_empty(&i915_file_priv->mm.request_list))
4967 list_del_init(i915_file_priv->mm.request_list.next);
4968 mutex_unlock(&dev->struct_mutex);
4969}
Chris Wilson31169712009-09-14 16:50:28 +01004970
Chris Wilson31169712009-09-14 16:50:28 +01004971static int
Chris Wilson1637ef42010-04-20 17:10:35 +01004972i915_gpu_is_active(struct drm_device *dev)
4973{
4974 drm_i915_private_t *dev_priv = dev->dev_private;
4975 int lists_empty;
4976
4977 spin_lock(&dev_priv->mm.active_list_lock);
4978 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
Zou Nan hai852835f2010-05-21 09:08:56 +08004979 list_empty(&dev_priv->render_ring.active_list);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004980 if (HAS_BSD(dev))
4981 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
Chris Wilson1637ef42010-04-20 17:10:35 +01004982 spin_unlock(&dev_priv->mm.active_list_lock);
4983
4984 return !lists_empty;
4985}
4986
4987static int
Chris Wilson31169712009-09-14 16:50:28 +01004988i915_gem_shrink(int nr_to_scan, gfp_t gfp_mask)
4989{
4990 drm_i915_private_t *dev_priv, *next_dev;
4991 struct drm_i915_gem_object *obj_priv, *next_obj;
4992 int cnt = 0;
4993 int would_deadlock = 1;
4994
4995 /* "fast-path" to count number of available objects */
4996 if (nr_to_scan == 0) {
4997 spin_lock(&shrink_list_lock);
4998 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4999 struct drm_device *dev = dev_priv->dev;
5000
5001 if (mutex_trylock(&dev->struct_mutex)) {
5002 list_for_each_entry(obj_priv,
5003 &dev_priv->mm.inactive_list,
5004 list)
5005 cnt++;
5006 mutex_unlock(&dev->struct_mutex);
5007 }
5008 }
5009 spin_unlock(&shrink_list_lock);
5010
5011 return (cnt / 100) * sysctl_vfs_cache_pressure;
5012 }
5013
5014 spin_lock(&shrink_list_lock);
5015
Chris Wilson1637ef42010-04-20 17:10:35 +01005016rescan:
Chris Wilson31169712009-09-14 16:50:28 +01005017 /* first scan for clean buffers */
5018 list_for_each_entry_safe(dev_priv, next_dev,
5019 &shrink_list, mm.shrink_list) {
5020 struct drm_device *dev = dev_priv->dev;
5021
5022 if (! mutex_trylock(&dev->struct_mutex))
5023 continue;
5024
5025 spin_unlock(&shrink_list_lock);
Zou Nan hai852835f2010-05-21 09:08:56 +08005026 i915_gem_retire_requests(dev, &dev_priv->render_ring);
Chris Wilson31169712009-09-14 16:50:28 +01005027
Zou Nan haid1b851f2010-05-21 09:08:57 +08005028 if (HAS_BSD(dev))
5029 i915_gem_retire_requests(dev, &dev_priv->bsd_ring);
5030
Chris Wilson31169712009-09-14 16:50:28 +01005031 list_for_each_entry_safe(obj_priv, next_obj,
5032 &dev_priv->mm.inactive_list,
5033 list) {
5034 if (i915_gem_object_is_purgeable(obj_priv)) {
Daniel Vettera8089e82010-04-09 19:05:09 +00005035 i915_gem_object_unbind(&obj_priv->base);
Chris Wilson31169712009-09-14 16:50:28 +01005036 if (--nr_to_scan <= 0)
5037 break;
5038 }
5039 }
5040
5041 spin_lock(&shrink_list_lock);
5042 mutex_unlock(&dev->struct_mutex);
5043
Chris Wilson963b4832009-09-20 23:03:54 +01005044 would_deadlock = 0;
5045
Chris Wilson31169712009-09-14 16:50:28 +01005046 if (nr_to_scan <= 0)
5047 break;
5048 }
5049
5050 /* second pass, evict/count anything still on the inactive list */
5051 list_for_each_entry_safe(dev_priv, next_dev,
5052 &shrink_list, mm.shrink_list) {
5053 struct drm_device *dev = dev_priv->dev;
5054
5055 if (! mutex_trylock(&dev->struct_mutex))
5056 continue;
5057
5058 spin_unlock(&shrink_list_lock);
5059
5060 list_for_each_entry_safe(obj_priv, next_obj,
5061 &dev_priv->mm.inactive_list,
5062 list) {
5063 if (nr_to_scan > 0) {
Daniel Vettera8089e82010-04-09 19:05:09 +00005064 i915_gem_object_unbind(&obj_priv->base);
Chris Wilson31169712009-09-14 16:50:28 +01005065 nr_to_scan--;
5066 } else
5067 cnt++;
5068 }
5069
5070 spin_lock(&shrink_list_lock);
5071 mutex_unlock(&dev->struct_mutex);
5072
5073 would_deadlock = 0;
5074 }
5075
Chris Wilson1637ef42010-04-20 17:10:35 +01005076 if (nr_to_scan) {
5077 int active = 0;
5078
5079 /*
5080 * We are desperate for pages, so as a last resort, wait
5081 * for the GPU to finish and discard whatever we can.
5082 * This has a dramatic impact to reduce the number of
5083 * OOM-killer events whilst running the GPU aggressively.
5084 */
5085 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5086 struct drm_device *dev = dev_priv->dev;
5087
5088 if (!mutex_trylock(&dev->struct_mutex))
5089 continue;
5090
5091 spin_unlock(&shrink_list_lock);
5092
5093 if (i915_gpu_is_active(dev)) {
5094 i915_gpu_idle(dev);
5095 active++;
5096 }
5097
5098 spin_lock(&shrink_list_lock);
5099 mutex_unlock(&dev->struct_mutex);
5100 }
5101
5102 if (active)
5103 goto rescan;
5104 }
5105
Chris Wilson31169712009-09-14 16:50:28 +01005106 spin_unlock(&shrink_list_lock);
5107
5108 if (would_deadlock)
5109 return -1;
5110 else if (cnt > 0)
5111 return (cnt / 100) * sysctl_vfs_cache_pressure;
5112 else
5113 return 0;
5114}
5115
5116static struct shrinker shrinker = {
5117 .shrink = i915_gem_shrink,
5118 .seeks = DEFAULT_SEEKS,
5119};
5120
5121__init void
5122i915_gem_shrinker_init(void)
5123{
5124 register_shrinker(&shrinker);
5125}
5126
5127__exit void
5128i915_gem_shrinker_exit(void)
5129{
5130 unregister_shrinker(&shrinker);
5131}