blob: 0ce28c71facca651d5580be3c374744869e205e3 [file] [log] [blame]
Eric Anholt673a3942008-07-30 12:06:12 -07001/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
Chris Wilson1c5d22f2009-08-25 11:15:50 +010032#include "i915_trace.h"
Jesse Barnes652c3932009-08-17 13:31:43 -070033#include "intel_drv.h"
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090034#include <linux/slab.h>
Eric Anholt673a3942008-07-30 12:06:12 -070035#include <linux/swap.h>
Jesse Barnes79e53942008-11-07 14:24:08 -080036#include <linux/pci.h>
Zhenyu Wangf8f235e2010-08-27 11:08:57 +080037#include <linux/intel-gtt.h>
Eric Anholt673a3942008-07-30 12:06:12 -070038
Daniel Vetter0108a3e2010-08-07 11:01:21 +010039static uint32_t i915_gem_get_gtt_alignment(struct drm_gem_object *obj);
Daniel Vetterba3d8d72010-02-11 22:37:04 +010040
41static int i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
42 bool pipelined);
Eric Anholte47c68e2008-11-14 13:35:19 -080043static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
44static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
Eric Anholte47c68e2008-11-14 13:35:19 -080045static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
46 int write);
47static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
48 uint64_t offset,
49 uint64_t size);
50static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
Chris Wilson2cf34d72010-09-14 13:03:28 +010051static int i915_gem_object_wait_rendering(struct drm_gem_object *obj,
52 bool interruptible);
Jesse Barnesde151cf2008-11-12 10:03:55 -080053static int i915_gem_object_bind_to_gtt(struct drm_gem_object *obj,
54 unsigned alignment);
Jesse Barnesde151cf2008-11-12 10:03:55 -080055static void i915_gem_clear_fence_reg(struct drm_gem_object *obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +100056static int i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
57 struct drm_i915_gem_pwrite *args,
58 struct drm_file *file_priv);
Chris Wilsonbe726152010-07-23 23:18:50 +010059static void i915_gem_free_object_tail(struct drm_gem_object *obj);
Eric Anholt673a3942008-07-30 12:06:12 -070060
Chris Wilson31169712009-09-14 16:50:28 +010061static LIST_HEAD(shrink_list);
62static DEFINE_SPINLOCK(shrink_list_lock);
63
Chris Wilson7d1c4802010-08-07 21:45:03 +010064static inline bool
65i915_gem_object_is_inactive(struct drm_i915_gem_object *obj_priv)
66{
67 return obj_priv->gtt_space &&
68 !obj_priv->active &&
69 obj_priv->pin_count == 0;
70}
71
Jesse Barnes79e53942008-11-07 14:24:08 -080072int i915_gem_do_init(struct drm_device *dev, unsigned long start,
73 unsigned long end)
74{
75 drm_i915_private_t *dev_priv = dev->dev_private;
76
77 if (start >= end ||
78 (start & (PAGE_SIZE - 1)) != 0 ||
79 (end & (PAGE_SIZE - 1)) != 0) {
80 return -EINVAL;
81 }
82
83 drm_mm_init(&dev_priv->mm.gtt_space, start,
84 end - start);
85
86 dev->gtt_total = (uint32_t) (end - start);
87
88 return 0;
89}
Keith Packard6dbe2772008-10-14 21:41:13 -070090
Eric Anholt673a3942008-07-30 12:06:12 -070091int
92i915_gem_init_ioctl(struct drm_device *dev, void *data,
93 struct drm_file *file_priv)
94{
Eric Anholt673a3942008-07-30 12:06:12 -070095 struct drm_i915_gem_init *args = data;
Jesse Barnes79e53942008-11-07 14:24:08 -080096 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -070097
98 mutex_lock(&dev->struct_mutex);
Jesse Barnes79e53942008-11-07 14:24:08 -080099 ret = i915_gem_do_init(dev, args->gtt_start, args->gtt_end);
Eric Anholt673a3942008-07-30 12:06:12 -0700100 mutex_unlock(&dev->struct_mutex);
101
Jesse Barnes79e53942008-11-07 14:24:08 -0800102 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700103}
104
Eric Anholt5a125c32008-10-22 21:40:13 -0700105int
106i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
107 struct drm_file *file_priv)
108{
Eric Anholt5a125c32008-10-22 21:40:13 -0700109 struct drm_i915_gem_get_aperture *args = data;
Eric Anholt5a125c32008-10-22 21:40:13 -0700110
111 if (!(dev->driver->driver_features & DRIVER_GEM))
112 return -ENODEV;
113
114 args->aper_size = dev->gtt_total;
Keith Packard2678d9d2008-11-20 22:54:54 -0800115 args->aper_available_size = (args->aper_size -
116 atomic_read(&dev->pin_memory));
Eric Anholt5a125c32008-10-22 21:40:13 -0700117
118 return 0;
119}
120
Eric Anholt673a3942008-07-30 12:06:12 -0700121
122/**
123 * Creates a new mm object and returns a handle to it.
124 */
125int
126i915_gem_create_ioctl(struct drm_device *dev, void *data,
127 struct drm_file *file_priv)
128{
129 struct drm_i915_gem_create *args = data;
130 struct drm_gem_object *obj;
Pekka Paalanena1a2d1d2009-08-23 12:40:55 +0300131 int ret;
132 u32 handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700133
134 args->size = roundup(args->size, PAGE_SIZE);
135
136 /* Allocate the new object */
Daniel Vetterac52bc52010-04-09 19:05:06 +0000137 obj = i915_gem_alloc_object(dev, args->size);
Eric Anholt673a3942008-07-30 12:06:12 -0700138 if (obj == NULL)
139 return -ENOMEM;
140
141 ret = drm_gem_handle_create(file_priv, obj, &handle);
Chris Wilson1dfd9752010-09-06 14:44:14 +0100142 if (ret) {
143 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700144 return ret;
Chris Wilson1dfd9752010-09-06 14:44:14 +0100145 }
146
147 /* Sink the floating reference from kref_init(handlecount) */
148 drm_gem_object_handle_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700149
150 args->handle = handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700151 return 0;
152}
153
Eric Anholt40123c12009-03-09 13:42:30 -0700154static inline int
Eric Anholteb014592009-03-10 11:44:52 -0700155fast_shmem_read(struct page **pages,
156 loff_t page_base, int page_offset,
157 char __user *data,
158 int length)
159{
160 char __iomem *vaddr;
Florian Mickler2bc43b52009-04-06 22:55:41 +0200161 int unwritten;
Eric Anholteb014592009-03-10 11:44:52 -0700162
163 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
164 if (vaddr == NULL)
165 return -ENOMEM;
Florian Mickler2bc43b52009-04-06 22:55:41 +0200166 unwritten = __copy_to_user_inatomic(data, vaddr + page_offset, length);
Eric Anholteb014592009-03-10 11:44:52 -0700167 kunmap_atomic(vaddr, KM_USER0);
168
Florian Mickler2bc43b52009-04-06 22:55:41 +0200169 if (unwritten)
170 return -EFAULT;
171
172 return 0;
Eric Anholteb014592009-03-10 11:44:52 -0700173}
174
Eric Anholt280b7132009-03-12 16:56:27 -0700175static int i915_gem_object_needs_bit17_swizzle(struct drm_gem_object *obj)
176{
177 drm_i915_private_t *dev_priv = obj->dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +0100178 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt280b7132009-03-12 16:56:27 -0700179
180 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
181 obj_priv->tiling_mode != I915_TILING_NONE;
182}
183
Chris Wilson99a03df2010-05-27 14:15:34 +0100184static inline void
Eric Anholt40123c12009-03-09 13:42:30 -0700185slow_shmem_copy(struct page *dst_page,
186 int dst_offset,
187 struct page *src_page,
188 int src_offset,
189 int length)
190{
191 char *dst_vaddr, *src_vaddr;
192
Chris Wilson99a03df2010-05-27 14:15:34 +0100193 dst_vaddr = kmap(dst_page);
194 src_vaddr = kmap(src_page);
Eric Anholt40123c12009-03-09 13:42:30 -0700195
196 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
197
Chris Wilson99a03df2010-05-27 14:15:34 +0100198 kunmap(src_page);
199 kunmap(dst_page);
Eric Anholt40123c12009-03-09 13:42:30 -0700200}
201
Chris Wilson99a03df2010-05-27 14:15:34 +0100202static inline void
Eric Anholt280b7132009-03-12 16:56:27 -0700203slow_shmem_bit17_copy(struct page *gpu_page,
204 int gpu_offset,
205 struct page *cpu_page,
206 int cpu_offset,
207 int length,
208 int is_read)
209{
210 char *gpu_vaddr, *cpu_vaddr;
211
212 /* Use the unswizzled path if this page isn't affected. */
213 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
214 if (is_read)
215 return slow_shmem_copy(cpu_page, cpu_offset,
216 gpu_page, gpu_offset, length);
217 else
218 return slow_shmem_copy(gpu_page, gpu_offset,
219 cpu_page, cpu_offset, length);
220 }
221
Chris Wilson99a03df2010-05-27 14:15:34 +0100222 gpu_vaddr = kmap(gpu_page);
223 cpu_vaddr = kmap(cpu_page);
Eric Anholt280b7132009-03-12 16:56:27 -0700224
225 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
226 * XORing with the other bits (A9 for Y, A9 and A10 for X)
227 */
228 while (length > 0) {
229 int cacheline_end = ALIGN(gpu_offset + 1, 64);
230 int this_length = min(cacheline_end - gpu_offset, length);
231 int swizzled_gpu_offset = gpu_offset ^ 64;
232
233 if (is_read) {
234 memcpy(cpu_vaddr + cpu_offset,
235 gpu_vaddr + swizzled_gpu_offset,
236 this_length);
237 } else {
238 memcpy(gpu_vaddr + swizzled_gpu_offset,
239 cpu_vaddr + cpu_offset,
240 this_length);
241 }
242 cpu_offset += this_length;
243 gpu_offset += this_length;
244 length -= this_length;
245 }
246
Chris Wilson99a03df2010-05-27 14:15:34 +0100247 kunmap(cpu_page);
248 kunmap(gpu_page);
Eric Anholt280b7132009-03-12 16:56:27 -0700249}
250
Eric Anholt673a3942008-07-30 12:06:12 -0700251/**
Eric Anholteb014592009-03-10 11:44:52 -0700252 * This is the fast shmem pread path, which attempts to copy_from_user directly
253 * from the backing pages of the object to the user's address space. On a
254 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
255 */
256static int
257i915_gem_shmem_pread_fast(struct drm_device *dev, struct drm_gem_object *obj,
258 struct drm_i915_gem_pread *args,
259 struct drm_file *file_priv)
260{
Daniel Vetter23010e42010-03-08 13:35:02 +0100261 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700262 ssize_t remain;
263 loff_t offset, page_base;
264 char __user *user_data;
265 int page_offset, page_length;
266 int ret;
267
268 user_data = (char __user *) (uintptr_t) args->data_ptr;
269 remain = args->size;
270
271 mutex_lock(&dev->struct_mutex);
272
Chris Wilson4bdadb92010-01-27 13:36:32 +0000273 ret = i915_gem_object_get_pages(obj, 0);
Eric Anholteb014592009-03-10 11:44:52 -0700274 if (ret != 0)
275 goto fail_unlock;
276
277 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
278 args->size);
279 if (ret != 0)
280 goto fail_put_pages;
281
Daniel Vetter23010e42010-03-08 13:35:02 +0100282 obj_priv = to_intel_bo(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700283 offset = args->offset;
284
285 while (remain > 0) {
286 /* Operation in this page
287 *
288 * page_base = page offset within aperture
289 * page_offset = offset within page
290 * page_length = bytes to copy for this page
291 */
292 page_base = (offset & ~(PAGE_SIZE-1));
293 page_offset = offset & (PAGE_SIZE-1);
294 page_length = remain;
295 if ((page_offset + remain) > PAGE_SIZE)
296 page_length = PAGE_SIZE - page_offset;
297
298 ret = fast_shmem_read(obj_priv->pages,
299 page_base, page_offset,
300 user_data, page_length);
301 if (ret)
302 goto fail_put_pages;
303
304 remain -= page_length;
305 user_data += page_length;
306 offset += page_length;
307 }
308
309fail_put_pages:
310 i915_gem_object_put_pages(obj);
311fail_unlock:
312 mutex_unlock(&dev->struct_mutex);
313
314 return ret;
315}
316
Chris Wilson07f73f62009-09-14 16:50:30 +0100317static int
318i915_gem_object_get_pages_or_evict(struct drm_gem_object *obj)
319{
320 int ret;
321
Chris Wilson4bdadb92010-01-27 13:36:32 +0000322 ret = i915_gem_object_get_pages(obj, __GFP_NORETRY | __GFP_NOWARN);
Chris Wilson07f73f62009-09-14 16:50:30 +0100323
324 /* If we've insufficient memory to map in the pages, attempt
325 * to make some space by throwing out some old buffers.
326 */
327 if (ret == -ENOMEM) {
328 struct drm_device *dev = obj->dev;
Chris Wilson07f73f62009-09-14 16:50:30 +0100329
Daniel Vetter0108a3e2010-08-07 11:01:21 +0100330 ret = i915_gem_evict_something(dev, obj->size,
331 i915_gem_get_gtt_alignment(obj));
Chris Wilson07f73f62009-09-14 16:50:30 +0100332 if (ret)
333 return ret;
334
Chris Wilson4bdadb92010-01-27 13:36:32 +0000335 ret = i915_gem_object_get_pages(obj, 0);
Chris Wilson07f73f62009-09-14 16:50:30 +0100336 }
337
338 return ret;
339}
340
Eric Anholteb014592009-03-10 11:44:52 -0700341/**
342 * This is the fallback shmem pread path, which allocates temporary storage
343 * in kernel space to copy_to_user into outside of the struct_mutex, so we
344 * can copy out of the object's backing pages while holding the struct mutex
345 * and not take page faults.
346 */
347static int
348i915_gem_shmem_pread_slow(struct drm_device *dev, struct drm_gem_object *obj,
349 struct drm_i915_gem_pread *args,
350 struct drm_file *file_priv)
351{
Daniel Vetter23010e42010-03-08 13:35:02 +0100352 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700353 struct mm_struct *mm = current->mm;
354 struct page **user_pages;
355 ssize_t remain;
356 loff_t offset, pinned_pages, i;
357 loff_t first_data_page, last_data_page, num_pages;
358 int shmem_page_index, shmem_page_offset;
359 int data_page_index, data_page_offset;
360 int page_length;
361 int ret;
362 uint64_t data_ptr = args->data_ptr;
Eric Anholt280b7132009-03-12 16:56:27 -0700363 int do_bit17_swizzling;
Eric Anholteb014592009-03-10 11:44:52 -0700364
365 remain = args->size;
366
367 /* Pin the user pages containing the data. We can't fault while
368 * holding the struct mutex, yet we want to hold it while
369 * dereferencing the user data.
370 */
371 first_data_page = data_ptr / PAGE_SIZE;
372 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
373 num_pages = last_data_page - first_data_page + 1;
374
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700375 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
Eric Anholteb014592009-03-10 11:44:52 -0700376 if (user_pages == NULL)
377 return -ENOMEM;
378
379 down_read(&mm->mmap_sem);
380 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
Eric Anholte5e9ecd2009-04-07 16:01:22 -0700381 num_pages, 1, 0, user_pages, NULL);
Eric Anholteb014592009-03-10 11:44:52 -0700382 up_read(&mm->mmap_sem);
383 if (pinned_pages < num_pages) {
384 ret = -EFAULT;
385 goto fail_put_user_pages;
386 }
387
Eric Anholt280b7132009-03-12 16:56:27 -0700388 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
389
Eric Anholteb014592009-03-10 11:44:52 -0700390 mutex_lock(&dev->struct_mutex);
391
Chris Wilson07f73f62009-09-14 16:50:30 +0100392 ret = i915_gem_object_get_pages_or_evict(obj);
393 if (ret)
Eric Anholteb014592009-03-10 11:44:52 -0700394 goto fail_unlock;
395
396 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
397 args->size);
398 if (ret != 0)
399 goto fail_put_pages;
400
Daniel Vetter23010e42010-03-08 13:35:02 +0100401 obj_priv = to_intel_bo(obj);
Eric Anholteb014592009-03-10 11:44:52 -0700402 offset = args->offset;
403
404 while (remain > 0) {
405 /* Operation in this page
406 *
407 * shmem_page_index = page number within shmem file
408 * shmem_page_offset = offset within page in shmem file
409 * data_page_index = page number in get_user_pages return
410 * data_page_offset = offset with data_page_index page.
411 * page_length = bytes to copy for this page
412 */
413 shmem_page_index = offset / PAGE_SIZE;
414 shmem_page_offset = offset & ~PAGE_MASK;
415 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
416 data_page_offset = data_ptr & ~PAGE_MASK;
417
418 page_length = remain;
419 if ((shmem_page_offset + page_length) > PAGE_SIZE)
420 page_length = PAGE_SIZE - shmem_page_offset;
421 if ((data_page_offset + page_length) > PAGE_SIZE)
422 page_length = PAGE_SIZE - data_page_offset;
423
Eric Anholt280b7132009-03-12 16:56:27 -0700424 if (do_bit17_swizzling) {
Chris Wilson99a03df2010-05-27 14:15:34 +0100425 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
Eric Anholt280b7132009-03-12 16:56:27 -0700426 shmem_page_offset,
Chris Wilson99a03df2010-05-27 14:15:34 +0100427 user_pages[data_page_index],
428 data_page_offset,
429 page_length,
430 1);
431 } else {
432 slow_shmem_copy(user_pages[data_page_index],
433 data_page_offset,
434 obj_priv->pages[shmem_page_index],
435 shmem_page_offset,
436 page_length);
Eric Anholt280b7132009-03-12 16:56:27 -0700437 }
Eric Anholteb014592009-03-10 11:44:52 -0700438
439 remain -= page_length;
440 data_ptr += page_length;
441 offset += page_length;
442 }
443
444fail_put_pages:
445 i915_gem_object_put_pages(obj);
446fail_unlock:
447 mutex_unlock(&dev->struct_mutex);
448fail_put_user_pages:
449 for (i = 0; i < pinned_pages; i++) {
450 SetPageDirty(user_pages[i]);
451 page_cache_release(user_pages[i]);
452 }
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700453 drm_free_large(user_pages);
Eric Anholteb014592009-03-10 11:44:52 -0700454
455 return ret;
456}
457
Eric Anholt673a3942008-07-30 12:06:12 -0700458/**
459 * Reads data from the object referenced by handle.
460 *
461 * On error, the contents of *data are undefined.
462 */
463int
464i915_gem_pread_ioctl(struct drm_device *dev, void *data,
465 struct drm_file *file_priv)
466{
467 struct drm_i915_gem_pread *args = data;
468 struct drm_gem_object *obj;
469 struct drm_i915_gem_object *obj_priv;
Eric Anholt673a3942008-07-30 12:06:12 -0700470 int ret;
471
472 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
473 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +0100474 return -ENOENT;
Daniel Vetter23010e42010-03-08 13:35:02 +0100475 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700476
477 /* Bounds check source.
478 *
479 * XXX: This could use review for overflow issues...
480 */
481 if (args->offset > obj->size || args->size > obj->size ||
482 args->offset + args->size > obj->size) {
Luca Barbieribc9025b2010-02-09 05:49:12 +0000483 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700484 return -EINVAL;
485 }
486
Eric Anholt280b7132009-03-12 16:56:27 -0700487 if (i915_gem_object_needs_bit17_swizzle(obj)) {
Eric Anholteb014592009-03-10 11:44:52 -0700488 ret = i915_gem_shmem_pread_slow(dev, obj, args, file_priv);
Eric Anholt280b7132009-03-12 16:56:27 -0700489 } else {
490 ret = i915_gem_shmem_pread_fast(dev, obj, args, file_priv);
491 if (ret != 0)
492 ret = i915_gem_shmem_pread_slow(dev, obj, args,
493 file_priv);
494 }
Eric Anholt673a3942008-07-30 12:06:12 -0700495
Luca Barbieribc9025b2010-02-09 05:49:12 +0000496 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700497
Eric Anholteb014592009-03-10 11:44:52 -0700498 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700499}
500
Keith Packard0839ccb2008-10-30 19:38:48 -0700501/* This is the fast write path which cannot handle
502 * page faults in the source data
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700503 */
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700504
Keith Packard0839ccb2008-10-30 19:38:48 -0700505static inline int
506fast_user_write(struct io_mapping *mapping,
507 loff_t page_base, int page_offset,
508 char __user *user_data,
509 int length)
510{
511 char *vaddr_atomic;
512 unsigned long unwritten;
513
Chris Wilsonfca3ec02010-08-04 14:34:24 +0100514 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base, KM_USER0);
Keith Packard0839ccb2008-10-30 19:38:48 -0700515 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
516 user_data, length);
Chris Wilsonfca3ec02010-08-04 14:34:24 +0100517 io_mapping_unmap_atomic(vaddr_atomic, KM_USER0);
Keith Packard0839ccb2008-10-30 19:38:48 -0700518 if (unwritten)
519 return -EFAULT;
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700520 return 0;
Keith Packard0839ccb2008-10-30 19:38:48 -0700521}
522
523/* Here's the write path which can sleep for
524 * page faults
525 */
526
Chris Wilsonab34c222010-05-27 14:15:35 +0100527static inline void
Eric Anholt3de09aa2009-03-09 09:42:23 -0700528slow_kernel_write(struct io_mapping *mapping,
529 loff_t gtt_base, int gtt_offset,
530 struct page *user_page, int user_offset,
531 int length)
Keith Packard0839ccb2008-10-30 19:38:48 -0700532{
Chris Wilsonab34c222010-05-27 14:15:35 +0100533 char __iomem *dst_vaddr;
534 char *src_vaddr;
Keith Packard0839ccb2008-10-30 19:38:48 -0700535
Chris Wilsonab34c222010-05-27 14:15:35 +0100536 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
537 src_vaddr = kmap(user_page);
538
539 memcpy_toio(dst_vaddr + gtt_offset,
540 src_vaddr + user_offset,
541 length);
542
543 kunmap(user_page);
544 io_mapping_unmap(dst_vaddr);
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700545}
546
Eric Anholt40123c12009-03-09 13:42:30 -0700547static inline int
548fast_shmem_write(struct page **pages,
549 loff_t page_base, int page_offset,
550 char __user *data,
551 int length)
552{
553 char __iomem *vaddr;
Dave Airlied0088772009-03-28 20:29:48 -0400554 unsigned long unwritten;
Eric Anholt40123c12009-03-09 13:42:30 -0700555
556 vaddr = kmap_atomic(pages[page_base >> PAGE_SHIFT], KM_USER0);
557 if (vaddr == NULL)
558 return -ENOMEM;
Dave Airlied0088772009-03-28 20:29:48 -0400559 unwritten = __copy_from_user_inatomic(vaddr + page_offset, data, length);
Eric Anholt40123c12009-03-09 13:42:30 -0700560 kunmap_atomic(vaddr, KM_USER0);
561
Dave Airlied0088772009-03-28 20:29:48 -0400562 if (unwritten)
563 return -EFAULT;
Eric Anholt40123c12009-03-09 13:42:30 -0700564 return 0;
565}
566
Eric Anholt3de09aa2009-03-09 09:42:23 -0700567/**
568 * This is the fast pwrite path, where we copy the data directly from the
569 * user into the GTT, uncached.
570 */
Eric Anholt673a3942008-07-30 12:06:12 -0700571static int
Eric Anholt3de09aa2009-03-09 09:42:23 -0700572i915_gem_gtt_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
573 struct drm_i915_gem_pwrite *args,
574 struct drm_file *file_priv)
Eric Anholt673a3942008-07-30 12:06:12 -0700575{
Daniel Vetter23010e42010-03-08 13:35:02 +0100576 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Keith Packard0839ccb2008-10-30 19:38:48 -0700577 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -0700578 ssize_t remain;
Keith Packard0839ccb2008-10-30 19:38:48 -0700579 loff_t offset, page_base;
Eric Anholt673a3942008-07-30 12:06:12 -0700580 char __user *user_data;
Keith Packard0839ccb2008-10-30 19:38:48 -0700581 int page_offset, page_length;
582 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700583
584 user_data = (char __user *) (uintptr_t) args->data_ptr;
585 remain = args->size;
586 if (!access_ok(VERIFY_READ, user_data, remain))
587 return -EFAULT;
588
589
590 mutex_lock(&dev->struct_mutex);
591 ret = i915_gem_object_pin(obj, 0);
592 if (ret) {
593 mutex_unlock(&dev->struct_mutex);
594 return ret;
595 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800596 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
Eric Anholt673a3942008-07-30 12:06:12 -0700597 if (ret)
598 goto fail;
599
Daniel Vetter23010e42010-03-08 13:35:02 +0100600 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700601 offset = obj_priv->gtt_offset + args->offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700602
603 while (remain > 0) {
604 /* Operation in this page
605 *
Keith Packard0839ccb2008-10-30 19:38:48 -0700606 * page_base = page offset within aperture
607 * page_offset = offset within page
608 * page_length = bytes to copy for this page
Eric Anholt673a3942008-07-30 12:06:12 -0700609 */
Keith Packard0839ccb2008-10-30 19:38:48 -0700610 page_base = (offset & ~(PAGE_SIZE-1));
611 page_offset = offset & (PAGE_SIZE-1);
612 page_length = remain;
613 if ((page_offset + remain) > PAGE_SIZE)
614 page_length = PAGE_SIZE - page_offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700615
Keith Packard0839ccb2008-10-30 19:38:48 -0700616 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
617 page_offset, user_data, page_length);
Eric Anholt673a3942008-07-30 12:06:12 -0700618
Keith Packard0839ccb2008-10-30 19:38:48 -0700619 /* If we get a fault while copying data, then (presumably) our
Eric Anholt3de09aa2009-03-09 09:42:23 -0700620 * source page isn't available. Return the error and we'll
621 * retry in the slow path.
Keith Packard0839ccb2008-10-30 19:38:48 -0700622 */
Eric Anholt3de09aa2009-03-09 09:42:23 -0700623 if (ret)
624 goto fail;
Eric Anholt673a3942008-07-30 12:06:12 -0700625
Keith Packard0839ccb2008-10-30 19:38:48 -0700626 remain -= page_length;
627 user_data += page_length;
628 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700629 }
Eric Anholt673a3942008-07-30 12:06:12 -0700630
631fail:
632 i915_gem_object_unpin(obj);
633 mutex_unlock(&dev->struct_mutex);
634
635 return ret;
636}
637
Eric Anholt3de09aa2009-03-09 09:42:23 -0700638/**
639 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
640 * the memory and maps it using kmap_atomic for copying.
641 *
642 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
643 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
644 */
Eric Anholt3043c602008-10-02 12:24:47 -0700645static int
Eric Anholt3de09aa2009-03-09 09:42:23 -0700646i915_gem_gtt_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
647 struct drm_i915_gem_pwrite *args,
648 struct drm_file *file_priv)
Eric Anholt673a3942008-07-30 12:06:12 -0700649{
Daniel Vetter23010e42010-03-08 13:35:02 +0100650 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700651 drm_i915_private_t *dev_priv = dev->dev_private;
652 ssize_t remain;
653 loff_t gtt_page_base, offset;
654 loff_t first_data_page, last_data_page, num_pages;
655 loff_t pinned_pages, i;
656 struct page **user_pages;
657 struct mm_struct *mm = current->mm;
658 int gtt_page_offset, data_page_offset, data_page_index, page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700659 int ret;
Eric Anholt3de09aa2009-03-09 09:42:23 -0700660 uint64_t data_ptr = args->data_ptr;
661
662 remain = args->size;
663
664 /* Pin the user pages containing the data. We can't fault while
665 * holding the struct mutex, and all of the pwrite implementations
666 * want to hold it while dereferencing the user data.
667 */
668 first_data_page = data_ptr / PAGE_SIZE;
669 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
670 num_pages = last_data_page - first_data_page + 1;
671
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700672 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
Eric Anholt3de09aa2009-03-09 09:42:23 -0700673 if (user_pages == NULL)
674 return -ENOMEM;
675
676 down_read(&mm->mmap_sem);
677 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
678 num_pages, 0, 0, user_pages, NULL);
679 up_read(&mm->mmap_sem);
680 if (pinned_pages < num_pages) {
681 ret = -EFAULT;
682 goto out_unpin_pages;
683 }
684
685 mutex_lock(&dev->struct_mutex);
686 ret = i915_gem_object_pin(obj, 0);
687 if (ret)
688 goto out_unlock;
689
690 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
691 if (ret)
692 goto out_unpin_object;
693
Daniel Vetter23010e42010-03-08 13:35:02 +0100694 obj_priv = to_intel_bo(obj);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700695 offset = obj_priv->gtt_offset + args->offset;
696
697 while (remain > 0) {
698 /* Operation in this page
699 *
700 * gtt_page_base = page offset within aperture
701 * gtt_page_offset = offset within page in aperture
702 * data_page_index = page number in get_user_pages return
703 * data_page_offset = offset with data_page_index page.
704 * page_length = bytes to copy for this page
705 */
706 gtt_page_base = offset & PAGE_MASK;
707 gtt_page_offset = offset & ~PAGE_MASK;
708 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
709 data_page_offset = data_ptr & ~PAGE_MASK;
710
711 page_length = remain;
712 if ((gtt_page_offset + page_length) > PAGE_SIZE)
713 page_length = PAGE_SIZE - gtt_page_offset;
714 if ((data_page_offset + page_length) > PAGE_SIZE)
715 page_length = PAGE_SIZE - data_page_offset;
716
Chris Wilsonab34c222010-05-27 14:15:35 +0100717 slow_kernel_write(dev_priv->mm.gtt_mapping,
718 gtt_page_base, gtt_page_offset,
719 user_pages[data_page_index],
720 data_page_offset,
721 page_length);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700722
723 remain -= page_length;
724 offset += page_length;
725 data_ptr += page_length;
726 }
727
728out_unpin_object:
729 i915_gem_object_unpin(obj);
730out_unlock:
731 mutex_unlock(&dev->struct_mutex);
732out_unpin_pages:
733 for (i = 0; i < pinned_pages; i++)
734 page_cache_release(user_pages[i]);
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700735 drm_free_large(user_pages);
Eric Anholt3de09aa2009-03-09 09:42:23 -0700736
737 return ret;
738}
739
Eric Anholt40123c12009-03-09 13:42:30 -0700740/**
741 * This is the fast shmem pwrite path, which attempts to directly
742 * copy_from_user into the kmapped pages backing the object.
743 */
Eric Anholt673a3942008-07-30 12:06:12 -0700744static int
Eric Anholt40123c12009-03-09 13:42:30 -0700745i915_gem_shmem_pwrite_fast(struct drm_device *dev, struct drm_gem_object *obj,
746 struct drm_i915_gem_pwrite *args,
747 struct drm_file *file_priv)
Eric Anholt673a3942008-07-30 12:06:12 -0700748{
Daniel Vetter23010e42010-03-08 13:35:02 +0100749 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt40123c12009-03-09 13:42:30 -0700750 ssize_t remain;
751 loff_t offset, page_base;
752 char __user *user_data;
753 int page_offset, page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700754 int ret;
Eric Anholt40123c12009-03-09 13:42:30 -0700755
756 user_data = (char __user *) (uintptr_t) args->data_ptr;
757 remain = args->size;
Eric Anholt673a3942008-07-30 12:06:12 -0700758
759 mutex_lock(&dev->struct_mutex);
760
Chris Wilson4bdadb92010-01-27 13:36:32 +0000761 ret = i915_gem_object_get_pages(obj, 0);
Eric Anholt40123c12009-03-09 13:42:30 -0700762 if (ret != 0)
763 goto fail_unlock;
764
Eric Anholte47c68e2008-11-14 13:35:19 -0800765 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
Eric Anholt40123c12009-03-09 13:42:30 -0700766 if (ret != 0)
767 goto fail_put_pages;
Eric Anholt673a3942008-07-30 12:06:12 -0700768
Daniel Vetter23010e42010-03-08 13:35:02 +0100769 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700770 offset = args->offset;
Eric Anholt40123c12009-03-09 13:42:30 -0700771 obj_priv->dirty = 1;
Eric Anholt673a3942008-07-30 12:06:12 -0700772
Eric Anholt40123c12009-03-09 13:42:30 -0700773 while (remain > 0) {
774 /* Operation in this page
775 *
776 * page_base = page offset within aperture
777 * page_offset = offset within page
778 * page_length = bytes to copy for this page
779 */
780 page_base = (offset & ~(PAGE_SIZE-1));
781 page_offset = offset & (PAGE_SIZE-1);
782 page_length = remain;
783 if ((page_offset + remain) > PAGE_SIZE)
784 page_length = PAGE_SIZE - page_offset;
785
786 ret = fast_shmem_write(obj_priv->pages,
787 page_base, page_offset,
788 user_data, page_length);
789 if (ret)
790 goto fail_put_pages;
791
792 remain -= page_length;
793 user_data += page_length;
794 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700795 }
796
Eric Anholt40123c12009-03-09 13:42:30 -0700797fail_put_pages:
798 i915_gem_object_put_pages(obj);
799fail_unlock:
Eric Anholt673a3942008-07-30 12:06:12 -0700800 mutex_unlock(&dev->struct_mutex);
801
Eric Anholt40123c12009-03-09 13:42:30 -0700802 return ret;
803}
804
805/**
806 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
807 * the memory and maps it using kmap_atomic for copying.
808 *
809 * This avoids taking mmap_sem for faulting on the user's address while the
810 * struct_mutex is held.
811 */
812static int
813i915_gem_shmem_pwrite_slow(struct drm_device *dev, struct drm_gem_object *obj,
814 struct drm_i915_gem_pwrite *args,
815 struct drm_file *file_priv)
816{
Daniel Vetter23010e42010-03-08 13:35:02 +0100817 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt40123c12009-03-09 13:42:30 -0700818 struct mm_struct *mm = current->mm;
819 struct page **user_pages;
820 ssize_t remain;
821 loff_t offset, pinned_pages, i;
822 loff_t first_data_page, last_data_page, num_pages;
823 int shmem_page_index, shmem_page_offset;
824 int data_page_index, data_page_offset;
825 int page_length;
826 int ret;
827 uint64_t data_ptr = args->data_ptr;
Eric Anholt280b7132009-03-12 16:56:27 -0700828 int do_bit17_swizzling;
Eric Anholt40123c12009-03-09 13:42:30 -0700829
830 remain = args->size;
831
832 /* Pin the user pages containing the data. We can't fault while
833 * holding the struct mutex, and all of the pwrite implementations
834 * want to hold it while dereferencing the user data.
835 */
836 first_data_page = data_ptr / PAGE_SIZE;
837 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
838 num_pages = last_data_page - first_data_page + 1;
839
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700840 user_pages = drm_calloc_large(num_pages, sizeof(struct page *));
Eric Anholt40123c12009-03-09 13:42:30 -0700841 if (user_pages == NULL)
842 return -ENOMEM;
843
844 down_read(&mm->mmap_sem);
845 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
846 num_pages, 0, 0, user_pages, NULL);
847 up_read(&mm->mmap_sem);
848 if (pinned_pages < num_pages) {
849 ret = -EFAULT;
850 goto fail_put_user_pages;
851 }
852
Eric Anholt280b7132009-03-12 16:56:27 -0700853 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
854
Eric Anholt40123c12009-03-09 13:42:30 -0700855 mutex_lock(&dev->struct_mutex);
856
Chris Wilson07f73f62009-09-14 16:50:30 +0100857 ret = i915_gem_object_get_pages_or_evict(obj);
858 if (ret)
Eric Anholt40123c12009-03-09 13:42:30 -0700859 goto fail_unlock;
860
861 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
862 if (ret != 0)
863 goto fail_put_pages;
864
Daniel Vetter23010e42010-03-08 13:35:02 +0100865 obj_priv = to_intel_bo(obj);
Eric Anholt40123c12009-03-09 13:42:30 -0700866 offset = args->offset;
867 obj_priv->dirty = 1;
868
869 while (remain > 0) {
870 /* Operation in this page
871 *
872 * shmem_page_index = page number within shmem file
873 * shmem_page_offset = offset within page in shmem file
874 * data_page_index = page number in get_user_pages return
875 * data_page_offset = offset with data_page_index page.
876 * page_length = bytes to copy for this page
877 */
878 shmem_page_index = offset / PAGE_SIZE;
879 shmem_page_offset = offset & ~PAGE_MASK;
880 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
881 data_page_offset = data_ptr & ~PAGE_MASK;
882
883 page_length = remain;
884 if ((shmem_page_offset + page_length) > PAGE_SIZE)
885 page_length = PAGE_SIZE - shmem_page_offset;
886 if ((data_page_offset + page_length) > PAGE_SIZE)
887 page_length = PAGE_SIZE - data_page_offset;
888
Eric Anholt280b7132009-03-12 16:56:27 -0700889 if (do_bit17_swizzling) {
Chris Wilson99a03df2010-05-27 14:15:34 +0100890 slow_shmem_bit17_copy(obj_priv->pages[shmem_page_index],
Eric Anholt280b7132009-03-12 16:56:27 -0700891 shmem_page_offset,
892 user_pages[data_page_index],
893 data_page_offset,
Chris Wilson99a03df2010-05-27 14:15:34 +0100894 page_length,
895 0);
896 } else {
897 slow_shmem_copy(obj_priv->pages[shmem_page_index],
898 shmem_page_offset,
899 user_pages[data_page_index],
900 data_page_offset,
901 page_length);
Eric Anholt280b7132009-03-12 16:56:27 -0700902 }
Eric Anholt40123c12009-03-09 13:42:30 -0700903
904 remain -= page_length;
905 data_ptr += page_length;
906 offset += page_length;
907 }
908
909fail_put_pages:
910 i915_gem_object_put_pages(obj);
911fail_unlock:
912 mutex_unlock(&dev->struct_mutex);
913fail_put_user_pages:
914 for (i = 0; i < pinned_pages; i++)
915 page_cache_release(user_pages[i]);
Jesse Barnes8e7d2b22009-05-08 16:13:25 -0700916 drm_free_large(user_pages);
Eric Anholt40123c12009-03-09 13:42:30 -0700917
918 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700919}
920
921/**
922 * Writes data to the object referenced by handle.
923 *
924 * On error, the contents of the buffer that were to be modified are undefined.
925 */
926int
927i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
928 struct drm_file *file_priv)
929{
930 struct drm_i915_gem_pwrite *args = data;
931 struct drm_gem_object *obj;
932 struct drm_i915_gem_object *obj_priv;
933 int ret = 0;
934
935 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
936 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +0100937 return -ENOENT;
Daniel Vetter23010e42010-03-08 13:35:02 +0100938 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700939
940 /* Bounds check destination.
941 *
942 * XXX: This could use review for overflow issues...
943 */
944 if (args->offset > obj->size || args->size > obj->size ||
945 args->offset + args->size > obj->size) {
Luca Barbieribc9025b2010-02-09 05:49:12 +0000946 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700947 return -EINVAL;
948 }
949
950 /* We can only do the GTT pwrite on untiled buffers, as otherwise
951 * it would end up going through the fenced access, and we'll get
952 * different detiling behavior between reading and writing.
953 * pread/pwrite currently are reading and writing from the CPU
954 * perspective, requiring manual detiling by the client.
955 */
Dave Airlie71acb5e2008-12-30 20:31:46 +1000956 if (obj_priv->phys_obj)
957 ret = i915_gem_phys_pwrite(dev, obj, args, file_priv);
958 else if (obj_priv->tiling_mode == I915_TILING_NONE &&
Chris Wilson9b8c4a02010-05-27 14:21:01 +0100959 dev->gtt_total != 0 &&
960 obj->write_domain != I915_GEM_DOMAIN_CPU) {
Eric Anholt3de09aa2009-03-09 09:42:23 -0700961 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file_priv);
962 if (ret == -EFAULT) {
963 ret = i915_gem_gtt_pwrite_slow(dev, obj, args,
964 file_priv);
965 }
Eric Anholt280b7132009-03-12 16:56:27 -0700966 } else if (i915_gem_object_needs_bit17_swizzle(obj)) {
967 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file_priv);
Eric Anholt40123c12009-03-09 13:42:30 -0700968 } else {
969 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file_priv);
970 if (ret == -EFAULT) {
971 ret = i915_gem_shmem_pwrite_slow(dev, obj, args,
972 file_priv);
973 }
974 }
Eric Anholt673a3942008-07-30 12:06:12 -0700975
976#if WATCH_PWRITE
977 if (ret)
978 DRM_INFO("pwrite failed %d\n", ret);
979#endif
980
Luca Barbieribc9025b2010-02-09 05:49:12 +0000981 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700982
983 return ret;
984}
985
986/**
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800987 * Called when user space prepares to use an object with the CPU, either
988 * through the mmap ioctl's mapping or a GTT mapping.
Eric Anholt673a3942008-07-30 12:06:12 -0700989 */
990int
991i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
992 struct drm_file *file_priv)
993{
Eric Anholta09ba7f2009-08-29 12:49:51 -0700994 struct drm_i915_private *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -0700995 struct drm_i915_gem_set_domain *args = data;
996 struct drm_gem_object *obj;
Jesse Barnes652c3932009-08-17 13:31:43 -0700997 struct drm_i915_gem_object *obj_priv;
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800998 uint32_t read_domains = args->read_domains;
999 uint32_t write_domain = args->write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -07001000 int ret;
1001
1002 if (!(dev->driver->driver_features & DRIVER_GEM))
1003 return -ENODEV;
1004
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001005 /* Only handle setting domains to types used by the CPU. */
Chris Wilson21d509e2009-06-06 09:46:02 +01001006 if (write_domain & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001007 return -EINVAL;
1008
Chris Wilson21d509e2009-06-06 09:46:02 +01001009 if (read_domains & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001010 return -EINVAL;
1011
1012 /* Having something in the write domain implies it's in the read
1013 * domain, and only that read domain. Enforce that in the request.
1014 */
1015 if (write_domain != 0 && read_domains != write_domain)
1016 return -EINVAL;
1017
Eric Anholt673a3942008-07-30 12:06:12 -07001018 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1019 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +01001020 return -ENOENT;
Daniel Vetter23010e42010-03-08 13:35:02 +01001021 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001022
1023 mutex_lock(&dev->struct_mutex);
Jesse Barnes652c3932009-08-17 13:31:43 -07001024
1025 intel_mark_busy(dev, obj);
1026
Eric Anholt673a3942008-07-30 12:06:12 -07001027#if WATCH_BUF
Krzysztof Halasacfd43c02009-06-20 00:31:28 +02001028 DRM_INFO("set_domain_ioctl %p(%zd), %08x %08x\n",
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001029 obj, obj->size, read_domains, write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07001030#endif
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001031 if (read_domains & I915_GEM_DOMAIN_GTT) {
1032 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
Eric Anholt02354392008-11-26 13:58:13 -08001033
Eric Anholta09ba7f2009-08-29 12:49:51 -07001034 /* Update the LRU on the fence for the CPU access that's
1035 * about to occur.
1036 */
1037 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
Daniel Vetter007cc8a2010-04-28 11:02:31 +02001038 struct drm_i915_fence_reg *reg =
1039 &dev_priv->fence_regs[obj_priv->fence_reg];
1040 list_move_tail(&reg->lru_list,
Eric Anholta09ba7f2009-08-29 12:49:51 -07001041 &dev_priv->mm.fence_list);
1042 }
1043
Eric Anholt02354392008-11-26 13:58:13 -08001044 /* Silently promote "you're not bound, there was nothing to do"
1045 * to success, since the client was just asking us to
1046 * make sure everything was done.
1047 */
1048 if (ret == -EINVAL)
1049 ret = 0;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001050 } else {
Eric Anholte47c68e2008-11-14 13:35:19 -08001051 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001052 }
1053
Chris Wilson7d1c4802010-08-07 21:45:03 +01001054 /* Maintain LRU order of "inactive" objects */
1055 if (ret == 0 && i915_gem_object_is_inactive(obj_priv))
1056 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1057
Eric Anholt673a3942008-07-30 12:06:12 -07001058 drm_gem_object_unreference(obj);
1059 mutex_unlock(&dev->struct_mutex);
1060 return ret;
1061}
1062
1063/**
1064 * Called when user space has done writes to this buffer
1065 */
1066int
1067i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1068 struct drm_file *file_priv)
1069{
1070 struct drm_i915_gem_sw_finish *args = data;
1071 struct drm_gem_object *obj;
1072 struct drm_i915_gem_object *obj_priv;
1073 int ret = 0;
1074
1075 if (!(dev->driver->driver_features & DRIVER_GEM))
1076 return -ENODEV;
1077
1078 mutex_lock(&dev->struct_mutex);
1079 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1080 if (obj == NULL) {
1081 mutex_unlock(&dev->struct_mutex);
Chris Wilsonbf79cb92010-08-04 14:19:46 +01001082 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07001083 }
1084
1085#if WATCH_BUF
Krzysztof Halasacfd43c02009-06-20 00:31:28 +02001086 DRM_INFO("%s: sw_finish %d (%p %zd)\n",
Eric Anholt673a3942008-07-30 12:06:12 -07001087 __func__, args->handle, obj, obj->size);
1088#endif
Daniel Vetter23010e42010-03-08 13:35:02 +01001089 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001090
1091 /* Pinned buffers may be scanout, so flush the cache */
Eric Anholte47c68e2008-11-14 13:35:19 -08001092 if (obj_priv->pin_count)
1093 i915_gem_object_flush_cpu_write_domain(obj);
1094
Eric Anholt673a3942008-07-30 12:06:12 -07001095 drm_gem_object_unreference(obj);
1096 mutex_unlock(&dev->struct_mutex);
1097 return ret;
1098}
1099
1100/**
1101 * Maps the contents of an object, returning the address it is mapped
1102 * into.
1103 *
1104 * While the mapping holds a reference on the contents of the object, it doesn't
1105 * imply a ref on the object itself.
1106 */
1107int
1108i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1109 struct drm_file *file_priv)
1110{
1111 struct drm_i915_gem_mmap *args = data;
1112 struct drm_gem_object *obj;
1113 loff_t offset;
1114 unsigned long addr;
1115
1116 if (!(dev->driver->driver_features & DRIVER_GEM))
1117 return -ENODEV;
1118
1119 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1120 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +01001121 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07001122
1123 offset = args->offset;
1124
1125 down_write(&current->mm->mmap_sem);
1126 addr = do_mmap(obj->filp, 0, args->size,
1127 PROT_READ | PROT_WRITE, MAP_SHARED,
1128 args->offset);
1129 up_write(&current->mm->mmap_sem);
Luca Barbieribc9025b2010-02-09 05:49:12 +00001130 drm_gem_object_unreference_unlocked(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001131 if (IS_ERR((void *)addr))
1132 return addr;
1133
1134 args->addr_ptr = (uint64_t) addr;
1135
1136 return 0;
1137}
1138
Jesse Barnesde151cf2008-11-12 10:03:55 -08001139/**
1140 * i915_gem_fault - fault a page into the GTT
1141 * vma: VMA in question
1142 * vmf: fault info
1143 *
1144 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1145 * from userspace. The fault handler takes care of binding the object to
1146 * the GTT (if needed), allocating and programming a fence register (again,
1147 * only if needed based on whether the old reg is still valid or the object
1148 * is tiled) and inserting a new PTE into the faulting process.
1149 *
1150 * Note that the faulting process may involve evicting existing objects
1151 * from the GTT and/or fence registers to make room. So performance may
1152 * suffer if the GTT working set is large or there are few fence registers
1153 * left.
1154 */
1155int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1156{
1157 struct drm_gem_object *obj = vma->vm_private_data;
1158 struct drm_device *dev = obj->dev;
Chris Wilson7d1c4802010-08-07 21:45:03 +01001159 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001160 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001161 pgoff_t page_offset;
1162 unsigned long pfn;
1163 int ret = 0;
Jesse Barnes0f973f22009-01-26 17:10:45 -08001164 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001165
1166 /* We don't use vmf->pgoff since that has the fake offset */
1167 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1168 PAGE_SHIFT;
1169
1170 /* Now bind it into the GTT if needed */
1171 mutex_lock(&dev->struct_mutex);
1172 if (!obj_priv->gtt_space) {
Chris Wilsone67b8ce2009-09-14 16:50:26 +01001173 ret = i915_gem_object_bind_to_gtt(obj, 0);
Chris Wilsonc7150892009-09-23 00:43:56 +01001174 if (ret)
1175 goto unlock;
Kristian Høgsberg07f4f3e2009-05-27 14:37:28 -04001176
Jesse Barnesde151cf2008-11-12 10:03:55 -08001177 ret = i915_gem_object_set_to_gtt_domain(obj, write);
Chris Wilsonc7150892009-09-23 00:43:56 +01001178 if (ret)
1179 goto unlock;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001180 }
1181
1182 /* Need a new fence register? */
Eric Anholta09ba7f2009-08-29 12:49:51 -07001183 if (obj_priv->tiling_mode != I915_TILING_NONE) {
Chris Wilson2cf34d72010-09-14 13:03:28 +01001184 ret = i915_gem_object_get_fence_reg(obj, true);
Chris Wilsonc7150892009-09-23 00:43:56 +01001185 if (ret)
1186 goto unlock;
Eric Anholtd9ddcb92009-01-27 10:33:49 -08001187 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08001188
Chris Wilson7d1c4802010-08-07 21:45:03 +01001189 if (i915_gem_object_is_inactive(obj_priv))
1190 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1191
Jesse Barnesde151cf2008-11-12 10:03:55 -08001192 pfn = ((dev->agp->base + obj_priv->gtt_offset) >> PAGE_SHIFT) +
1193 page_offset;
1194
1195 /* Finally, remap it using the new GTT offset */
1196 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
Chris Wilsonc7150892009-09-23 00:43:56 +01001197unlock:
Jesse Barnesde151cf2008-11-12 10:03:55 -08001198 mutex_unlock(&dev->struct_mutex);
1199
1200 switch (ret) {
Chris Wilsonc7150892009-09-23 00:43:56 +01001201 case 0:
1202 case -ERESTARTSYS:
1203 return VM_FAULT_NOPAGE;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001204 case -ENOMEM:
1205 case -EAGAIN:
1206 return VM_FAULT_OOM;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001207 default:
Chris Wilsonc7150892009-09-23 00:43:56 +01001208 return VM_FAULT_SIGBUS;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001209 }
1210}
1211
1212/**
1213 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1214 * @obj: obj in question
1215 *
1216 * GEM memory mapping works by handing back to userspace a fake mmap offset
1217 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1218 * up the object based on the offset and sets up the various memory mapping
1219 * structures.
1220 *
1221 * This routine allocates and attaches a fake offset for @obj.
1222 */
1223static int
1224i915_gem_create_mmap_offset(struct drm_gem_object *obj)
1225{
1226 struct drm_device *dev = obj->dev;
1227 struct drm_gem_mm *mm = dev->mm_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001228 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001229 struct drm_map_list *list;
Benjamin Herrenschmidtf77d3902009-02-02 16:55:46 +11001230 struct drm_local_map *map;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001231 int ret = 0;
1232
1233 /* Set the object up for mmap'ing */
1234 list = &obj->map_list;
Eric Anholt9a298b22009-03-24 12:23:04 -07001235 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001236 if (!list->map)
1237 return -ENOMEM;
1238
1239 map = list->map;
1240 map->type = _DRM_GEM;
1241 map->size = obj->size;
1242 map->handle = obj;
1243
1244 /* Get a DRM GEM mmap offset allocated... */
1245 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1246 obj->size / PAGE_SIZE, 0, 0);
1247 if (!list->file_offset_node) {
1248 DRM_ERROR("failed to allocate offset for bo %d\n", obj->name);
Chris Wilson9e0ae532010-09-21 15:05:24 +01001249 ret = -ENOSPC;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001250 goto out_free_list;
1251 }
1252
1253 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1254 obj->size / PAGE_SIZE, 0);
1255 if (!list->file_offset_node) {
1256 ret = -ENOMEM;
1257 goto out_free_list;
1258 }
1259
1260 list->hash.key = list->file_offset_node->start;
Chris Wilson9e0ae532010-09-21 15:05:24 +01001261 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1262 if (ret) {
Jesse Barnesde151cf2008-11-12 10:03:55 -08001263 DRM_ERROR("failed to add to map hash\n");
1264 goto out_free_mm;
1265 }
1266
1267 /* By now we should be all set, any drm_mmap request on the offset
1268 * below will get to our mmap & fault handler */
1269 obj_priv->mmap_offset = ((uint64_t) list->hash.key) << PAGE_SHIFT;
1270
1271 return 0;
1272
1273out_free_mm:
1274 drm_mm_put_block(list->file_offset_node);
1275out_free_list:
Eric Anholt9a298b22009-03-24 12:23:04 -07001276 kfree(list->map);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001277
1278 return ret;
1279}
1280
Chris Wilson901782b2009-07-10 08:18:50 +01001281/**
1282 * i915_gem_release_mmap - remove physical page mappings
1283 * @obj: obj in question
1284 *
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02001285 * Preserve the reservation of the mmapping with the DRM core code, but
Chris Wilson901782b2009-07-10 08:18:50 +01001286 * relinquish ownership of the pages back to the system.
1287 *
1288 * It is vital that we remove the page mapping if we have mapped a tiled
1289 * object through the GTT and then lose the fence register due to
1290 * resource pressure. Similarly if the object has been moved out of the
1291 * aperture, than pages mapped into userspace must be revoked. Removing the
1292 * mapping will then trigger a page fault on the next user access, allowing
1293 * fixup by i915_gem_fault().
1294 */
Eric Anholtd05ca302009-07-10 13:02:26 -07001295void
Chris Wilson901782b2009-07-10 08:18:50 +01001296i915_gem_release_mmap(struct drm_gem_object *obj)
1297{
1298 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001299 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson901782b2009-07-10 08:18:50 +01001300
1301 if (dev->dev_mapping)
1302 unmap_mapping_range(dev->dev_mapping,
1303 obj_priv->mmap_offset, obj->size, 1);
1304}
1305
Jesse Barnesab00b3e2009-02-11 14:01:46 -08001306static void
1307i915_gem_free_mmap_offset(struct drm_gem_object *obj)
1308{
1309 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001310 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesab00b3e2009-02-11 14:01:46 -08001311 struct drm_gem_mm *mm = dev->mm_private;
1312 struct drm_map_list *list;
1313
1314 list = &obj->map_list;
1315 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1316
1317 if (list->file_offset_node) {
1318 drm_mm_put_block(list->file_offset_node);
1319 list->file_offset_node = NULL;
1320 }
1321
1322 if (list->map) {
Eric Anholt9a298b22009-03-24 12:23:04 -07001323 kfree(list->map);
Jesse Barnesab00b3e2009-02-11 14:01:46 -08001324 list->map = NULL;
1325 }
1326
1327 obj_priv->mmap_offset = 0;
1328}
1329
Jesse Barnesde151cf2008-11-12 10:03:55 -08001330/**
1331 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1332 * @obj: object to check
1333 *
1334 * Return the required GTT alignment for an object, taking into account
1335 * potential fence register mapping if needed.
1336 */
1337static uint32_t
1338i915_gem_get_gtt_alignment(struct drm_gem_object *obj)
1339{
1340 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001341 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001342 int start, i;
1343
1344 /*
1345 * Minimum alignment is 4k (GTT page size), but might be greater
1346 * if a fence register is needed for the object.
1347 */
Chris Wilsona6c45cf2010-09-17 00:32:17 +01001348 if (INTEL_INFO(dev)->gen >= 4 || obj_priv->tiling_mode == I915_TILING_NONE)
Jesse Barnesde151cf2008-11-12 10:03:55 -08001349 return 4096;
1350
1351 /*
1352 * Previous chips need to be aligned to the size of the smallest
1353 * fence register that can contain the object.
1354 */
Chris Wilsona6c45cf2010-09-17 00:32:17 +01001355 if (INTEL_INFO(dev)->gen == 3)
Jesse Barnesde151cf2008-11-12 10:03:55 -08001356 start = 1024*1024;
1357 else
1358 start = 512*1024;
1359
1360 for (i = start; i < obj->size; i <<= 1)
1361 ;
1362
1363 return i;
1364}
1365
1366/**
1367 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1368 * @dev: DRM device
1369 * @data: GTT mapping ioctl data
1370 * @file_priv: GEM object info
1371 *
1372 * Simply returns the fake offset to userspace so it can mmap it.
1373 * The mmap call will end up in drm_gem_mmap(), which will set things
1374 * up so we can get faults in the handler above.
1375 *
1376 * The fault handler will take care of binding the object into the GTT
1377 * (since it may have been evicted to make room for something), allocating
1378 * a fence register, and mapping the appropriate aperture address into
1379 * userspace.
1380 */
1381int
1382i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1383 struct drm_file *file_priv)
1384{
1385 struct drm_i915_gem_mmap_gtt *args = data;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001386 struct drm_gem_object *obj;
1387 struct drm_i915_gem_object *obj_priv;
1388 int ret;
1389
1390 if (!(dev->driver->driver_features & DRIVER_GEM))
1391 return -ENODEV;
1392
1393 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
1394 if (obj == NULL)
Chris Wilsonbf79cb92010-08-04 14:19:46 +01001395 return -ENOENT;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001396
1397 mutex_lock(&dev->struct_mutex);
1398
Daniel Vetter23010e42010-03-08 13:35:02 +01001399 obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001400
Chris Wilsonab182822009-09-22 18:46:17 +01001401 if (obj_priv->madv != I915_MADV_WILLNEED) {
1402 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1403 drm_gem_object_unreference(obj);
1404 mutex_unlock(&dev->struct_mutex);
1405 return -EINVAL;
1406 }
1407
1408
Jesse Barnesde151cf2008-11-12 10:03:55 -08001409 if (!obj_priv->mmap_offset) {
1410 ret = i915_gem_create_mmap_offset(obj);
Chris Wilson13af1062009-02-11 14:26:31 +00001411 if (ret) {
1412 drm_gem_object_unreference(obj);
1413 mutex_unlock(&dev->struct_mutex);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001414 return ret;
Chris Wilson13af1062009-02-11 14:26:31 +00001415 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08001416 }
1417
1418 args->offset = obj_priv->mmap_offset;
1419
Jesse Barnesde151cf2008-11-12 10:03:55 -08001420 /*
1421 * Pull it into the GTT so that we have a page list (makes the
1422 * initial fault faster and any subsequent flushing possible).
1423 */
1424 if (!obj_priv->agp_mem) {
Chris Wilsone67b8ce2009-09-14 16:50:26 +01001425 ret = i915_gem_object_bind_to_gtt(obj, 0);
Jesse Barnesde151cf2008-11-12 10:03:55 -08001426 if (ret) {
1427 drm_gem_object_unreference(obj);
1428 mutex_unlock(&dev->struct_mutex);
1429 return ret;
1430 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08001431 }
1432
1433 drm_gem_object_unreference(obj);
1434 mutex_unlock(&dev->struct_mutex);
1435
1436 return 0;
1437}
1438
Ben Gamari6911a9b2009-04-02 11:24:54 -07001439void
Eric Anholt856fa192009-03-19 14:10:50 -07001440i915_gem_object_put_pages(struct drm_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07001441{
Daniel Vetter23010e42010-03-08 13:35:02 +01001442 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001443 int page_count = obj->size / PAGE_SIZE;
1444 int i;
1445
Eric Anholt856fa192009-03-19 14:10:50 -07001446 BUG_ON(obj_priv->pages_refcount == 0);
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001447 BUG_ON(obj_priv->madv == __I915_MADV_PURGED);
Eric Anholt856fa192009-03-19 14:10:50 -07001448
1449 if (--obj_priv->pages_refcount != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07001450 return;
1451
Eric Anholt280b7132009-03-12 16:56:27 -07001452 if (obj_priv->tiling_mode != I915_TILING_NONE)
1453 i915_gem_object_save_bit_17_swizzle(obj);
1454
Chris Wilson3ef94da2009-09-14 16:50:29 +01001455 if (obj_priv->madv == I915_MADV_DONTNEED)
Chris Wilson13a05fd2009-09-20 23:03:19 +01001456 obj_priv->dirty = 0;
Chris Wilson3ef94da2009-09-14 16:50:29 +01001457
1458 for (i = 0; i < page_count; i++) {
Chris Wilson3ef94da2009-09-14 16:50:29 +01001459 if (obj_priv->dirty)
1460 set_page_dirty(obj_priv->pages[i]);
1461
1462 if (obj_priv->madv == I915_MADV_WILLNEED)
Eric Anholt856fa192009-03-19 14:10:50 -07001463 mark_page_accessed(obj_priv->pages[i]);
Chris Wilson3ef94da2009-09-14 16:50:29 +01001464
1465 page_cache_release(obj_priv->pages[i]);
1466 }
Eric Anholt673a3942008-07-30 12:06:12 -07001467 obj_priv->dirty = 0;
1468
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07001469 drm_free_large(obj_priv->pages);
Eric Anholt856fa192009-03-19 14:10:50 -07001470 obj_priv->pages = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07001471}
1472
Daniel Vettere35a41d2010-02-11 22:13:59 +01001473static uint32_t
Daniel Vettera6910432010-02-02 17:08:37 +01001474i915_gem_next_request_seqno(struct drm_device *dev,
1475 struct intel_ring_buffer *ring)
Daniel Vettere35a41d2010-02-11 22:13:59 +01001476{
1477 drm_i915_private_t *dev_priv = dev->dev_private;
1478
Daniel Vettera6910432010-02-02 17:08:37 +01001479 ring->outstanding_lazy_request = true;
1480
Daniel Vettere35a41d2010-02-11 22:13:59 +01001481 return dev_priv->next_seqno;
1482}
1483
Eric Anholt673a3942008-07-30 12:06:12 -07001484static void
Daniel Vetter617dbe22010-02-11 22:16:02 +01001485i915_gem_object_move_to_active(struct drm_gem_object *obj,
Zou Nan hai852835f2010-05-21 09:08:56 +08001486 struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001487{
1488 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001489 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Daniel Vetter617dbe22010-02-11 22:16:02 +01001490 uint32_t seqno = i915_gem_next_request_seqno(dev, ring);
1491
Zou Nan hai852835f2010-05-21 09:08:56 +08001492 BUG_ON(ring == NULL);
1493 obj_priv->ring = ring;
Eric Anholt673a3942008-07-30 12:06:12 -07001494
1495 /* Add a reference if we're newly entering the active list. */
1496 if (!obj_priv->active) {
1497 drm_gem_object_reference(obj);
1498 obj_priv->active = 1;
1499 }
Daniel Vettere35a41d2010-02-11 22:13:59 +01001500
Eric Anholt673a3942008-07-30 12:06:12 -07001501 /* Move from whatever list we were on to the tail of execution. */
Zou Nan hai852835f2010-05-21 09:08:56 +08001502 list_move_tail(&obj_priv->list, &ring->active_list);
Eric Anholtce44b0e2008-11-06 16:00:31 -08001503 obj_priv->last_rendering_seqno = seqno;
Eric Anholt673a3942008-07-30 12:06:12 -07001504}
1505
Eric Anholtce44b0e2008-11-06 16:00:31 -08001506static void
1507i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
1508{
1509 struct drm_device *dev = obj->dev;
1510 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001511 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholtce44b0e2008-11-06 16:00:31 -08001512
1513 BUG_ON(!obj_priv->active);
1514 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
1515 obj_priv->last_rendering_seqno = 0;
1516}
Eric Anholt673a3942008-07-30 12:06:12 -07001517
Chris Wilson963b4832009-09-20 23:03:54 +01001518/* Immediately discard the backing storage */
1519static void
1520i915_gem_object_truncate(struct drm_gem_object *obj)
1521{
Daniel Vetter23010e42010-03-08 13:35:02 +01001522 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001523 struct inode *inode;
Chris Wilson963b4832009-09-20 23:03:54 +01001524
Chris Wilsonae9fed62010-08-07 11:01:30 +01001525 /* Our goal here is to return as much of the memory as
1526 * is possible back to the system as we are called from OOM.
1527 * To do this we must instruct the shmfs to drop all of its
1528 * backing pages, *now*. Here we mirror the actions taken
1529 * when by shmem_delete_inode() to release the backing store.
1530 */
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001531 inode = obj->filp->f_path.dentry->d_inode;
Chris Wilsonae9fed62010-08-07 11:01:30 +01001532 truncate_inode_pages(inode->i_mapping, 0);
1533 if (inode->i_op->truncate_range)
1534 inode->i_op->truncate_range(inode, 0, (loff_t)-1);
Chris Wilsonbb6baf72009-09-22 14:24:13 +01001535
1536 obj_priv->madv = __I915_MADV_PURGED;
Chris Wilson963b4832009-09-20 23:03:54 +01001537}
1538
1539static inline int
1540i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj_priv)
1541{
1542 return obj_priv->madv == I915_MADV_DONTNEED;
1543}
1544
Eric Anholt673a3942008-07-30 12:06:12 -07001545static void
1546i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
1547{
1548 struct drm_device *dev = obj->dev;
1549 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01001550 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001551
1552 i915_verify_inactive(dev, __FILE__, __LINE__);
1553 if (obj_priv->pin_count != 0)
Chris Wilsonf13d3f72010-09-20 17:36:15 +01001554 list_move_tail(&obj_priv->list, &dev_priv->mm.pinned_list);
Eric Anholt673a3942008-07-30 12:06:12 -07001555 else
1556 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
1557
Daniel Vetter99fcb762010-02-07 16:20:18 +01001558 BUG_ON(!list_empty(&obj_priv->gpu_write_list));
1559
Eric Anholtce44b0e2008-11-06 16:00:31 -08001560 obj_priv->last_rendering_seqno = 0;
Zou Nan hai852835f2010-05-21 09:08:56 +08001561 obj_priv->ring = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07001562 if (obj_priv->active) {
1563 obj_priv->active = 0;
1564 drm_gem_object_unreference(obj);
1565 }
1566 i915_verify_inactive(dev, __FILE__, __LINE__);
1567}
1568
Chris Wilson92204342010-09-18 11:02:01 +01001569static void
Daniel Vetter63560392010-02-19 11:51:59 +01001570i915_gem_process_flushing_list(struct drm_device *dev,
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01001571 uint32_t flush_domains,
Zou Nan hai852835f2010-05-21 09:08:56 +08001572 struct intel_ring_buffer *ring)
Daniel Vetter63560392010-02-19 11:51:59 +01001573{
1574 drm_i915_private_t *dev_priv = dev->dev_private;
1575 struct drm_i915_gem_object *obj_priv, *next;
1576
1577 list_for_each_entry_safe(obj_priv, next,
1578 &dev_priv->mm.gpu_write_list,
1579 gpu_write_list) {
Daniel Vettera8089e82010-04-09 19:05:09 +00001580 struct drm_gem_object *obj = &obj_priv->base;
Daniel Vetter63560392010-02-19 11:51:59 +01001581
Chris Wilson2b6efaa2010-09-14 17:04:02 +01001582 if (obj->write_domain & flush_domains &&
1583 obj_priv->ring == ring) {
Daniel Vetter63560392010-02-19 11:51:59 +01001584 uint32_t old_write_domain = obj->write_domain;
1585
1586 obj->write_domain = 0;
1587 list_del_init(&obj_priv->gpu_write_list);
Daniel Vetter617dbe22010-02-11 22:16:02 +01001588 i915_gem_object_move_to_active(obj, ring);
Daniel Vetter63560392010-02-19 11:51:59 +01001589
1590 /* update the fence lru list */
Daniel Vetter007cc8a2010-04-28 11:02:31 +02001591 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
1592 struct drm_i915_fence_reg *reg =
1593 &dev_priv->fence_regs[obj_priv->fence_reg];
1594 list_move_tail(&reg->lru_list,
Daniel Vetter63560392010-02-19 11:51:59 +01001595 &dev_priv->mm.fence_list);
Daniel Vetter007cc8a2010-04-28 11:02:31 +02001596 }
Daniel Vetter63560392010-02-19 11:51:59 +01001597
1598 trace_i915_gem_object_change_domain(obj,
1599 obj->read_domains,
1600 old_write_domain);
1601 }
1602 }
1603}
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001604
Daniel Vetter5a5a0c62009-09-15 22:57:36 +02001605uint32_t
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01001606i915_add_request(struct drm_device *dev,
1607 struct drm_file *file_priv,
Chris Wilson8dc5d142010-08-12 12:36:12 +01001608 struct drm_i915_gem_request *request,
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01001609 struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001610{
1611 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholtb9624422009-06-03 07:27:35 +00001612 struct drm_i915_file_private *i915_file_priv = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07001613 uint32_t seqno;
1614 int was_empty;
Eric Anholt673a3942008-07-30 12:06:12 -07001615
Eric Anholtb9624422009-06-03 07:27:35 +00001616 if (file_priv != NULL)
1617 i915_file_priv = file_priv->driver_priv;
1618
Chris Wilson8dc5d142010-08-12 12:36:12 +01001619 if (request == NULL) {
1620 request = kzalloc(sizeof(*request), GFP_KERNEL);
1621 if (request == NULL)
1622 return 0;
1623 }
Eric Anholt673a3942008-07-30 12:06:12 -07001624
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01001625 seqno = ring->add_request(dev, ring, file_priv, 0);
Eric Anholt673a3942008-07-30 12:06:12 -07001626
1627 request->seqno = seqno;
Zou Nan hai852835f2010-05-21 09:08:56 +08001628 request->ring = ring;
Eric Anholt673a3942008-07-30 12:06:12 -07001629 request->emitted_jiffies = jiffies;
Zou Nan hai852835f2010-05-21 09:08:56 +08001630 was_empty = list_empty(&ring->request_list);
1631 list_add_tail(&request->list, &ring->request_list);
1632
Eric Anholtb9624422009-06-03 07:27:35 +00001633 if (i915_file_priv) {
1634 list_add_tail(&request->client_list,
1635 &i915_file_priv->mm.request_list);
1636 } else {
1637 INIT_LIST_HEAD(&request->client_list);
1638 }
Eric Anholt673a3942008-07-30 12:06:12 -07001639
Ben Gamarif65d9422009-09-14 17:48:44 -04001640 if (!dev_priv->mm.suspended) {
Chris Wilsonb3b079d2010-09-13 23:44:34 +01001641 mod_timer(&dev_priv->hangcheck_timer,
1642 jiffies + msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
Ben Gamarif65d9422009-09-14 17:48:44 -04001643 if (was_empty)
Chris Wilsonb3b079d2010-09-13 23:44:34 +01001644 queue_delayed_work(dev_priv->wq,
1645 &dev_priv->mm.retire_work, HZ);
Ben Gamarif65d9422009-09-14 17:48:44 -04001646 }
Eric Anholt673a3942008-07-30 12:06:12 -07001647 return seqno;
1648}
1649
1650/**
1651 * Command execution barrier
1652 *
1653 * Ensures that all commands in the ring are finished
1654 * before signalling the CPU
1655 */
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01001656static void
Zou Nan hai852835f2010-05-21 09:08:56 +08001657i915_retire_commands(struct drm_device *dev, struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001658{
Eric Anholt673a3942008-07-30 12:06:12 -07001659 uint32_t flush_domains = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07001660
1661 /* The sampler always gets flushed on i965 (sigh) */
Chris Wilsona6c45cf2010-09-17 00:32:17 +01001662 if (INTEL_INFO(dev)->gen >= 4)
Eric Anholt673a3942008-07-30 12:06:12 -07001663 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
Zou Nan hai852835f2010-05-21 09:08:56 +08001664
1665 ring->flush(dev, ring,
1666 I915_GEM_DOMAIN_COMMAND, flush_domains);
Eric Anholt673a3942008-07-30 12:06:12 -07001667}
1668
1669/**
Eric Anholt673a3942008-07-30 12:06:12 -07001670 * Returns true if seq1 is later than seq2.
1671 */
Ben Gamari22be1722009-09-14 17:48:43 -04001672bool
Eric Anholt673a3942008-07-30 12:06:12 -07001673i915_seqno_passed(uint32_t seq1, uint32_t seq2)
1674{
1675 return (int32_t)(seq1 - seq2) >= 0;
1676}
1677
1678uint32_t
Zou Nan hai852835f2010-05-21 09:08:56 +08001679i915_get_gem_seqno(struct drm_device *dev,
Zou Nan haid1b851f2010-05-21 09:08:57 +08001680 struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001681{
Zou Nan hai852835f2010-05-21 09:08:56 +08001682 return ring->get_gem_seqno(dev, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07001683}
1684
Chris Wilsondfaae392010-09-22 10:31:52 +01001685static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1686 struct intel_ring_buffer *ring)
Chris Wilson9375e442010-09-19 12:21:28 +01001687{
Chris Wilsondfaae392010-09-22 10:31:52 +01001688 while (!list_empty(&ring->request_list)) {
1689 struct drm_i915_gem_request *request;
Chris Wilson9375e442010-09-19 12:21:28 +01001690
Chris Wilsondfaae392010-09-22 10:31:52 +01001691 request = list_first_entry(&ring->request_list,
1692 struct drm_i915_gem_request,
1693 list);
1694
1695 list_del(&request->list);
1696 list_del(&request->client_list);
1697 kfree(request);
1698 }
1699
1700 while (!list_empty(&ring->active_list)) {
Chris Wilson9375e442010-09-19 12:21:28 +01001701 struct drm_i915_gem_object *obj_priv;
1702
Chris Wilsondfaae392010-09-22 10:31:52 +01001703 obj_priv = list_first_entry(&ring->active_list,
1704 struct drm_i915_gem_object,
1705 list);
1706
1707 obj_priv->base.write_domain = 0;
1708 list_del_init(&obj_priv->gpu_write_list);
1709 i915_gem_object_move_to_inactive(&obj_priv->base);
1710 }
1711}
1712
1713void i915_gem_reset_lists(struct drm_device *dev)
1714{
1715 struct drm_i915_private *dev_priv = dev->dev_private;
1716 struct drm_i915_gem_object *obj_priv;
1717
1718 i915_gem_reset_ring_lists(dev_priv, &dev_priv->render_ring);
1719 if (HAS_BSD(dev))
1720 i915_gem_reset_ring_lists(dev_priv, &dev_priv->bsd_ring);
1721
1722 /* Remove anything from the flushing lists. The GPU cache is likely
1723 * to be lost on reset along with the data, so simply move the
1724 * lost bo to the inactive list.
1725 */
1726 while (!list_empty(&dev_priv->mm.flushing_list)) {
Chris Wilson9375e442010-09-19 12:21:28 +01001727 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1728 struct drm_i915_gem_object,
1729 list);
1730
1731 obj_priv->base.write_domain = 0;
Chris Wilsondfaae392010-09-22 10:31:52 +01001732 list_del_init(&obj_priv->gpu_write_list);
Chris Wilson9375e442010-09-19 12:21:28 +01001733 i915_gem_object_move_to_inactive(&obj_priv->base);
1734 }
Chris Wilson9375e442010-09-19 12:21:28 +01001735
Chris Wilsondfaae392010-09-22 10:31:52 +01001736 /* Move everything out of the GPU domains to ensure we do any
1737 * necessary invalidation upon reuse.
1738 */
Chris Wilson77f01232010-09-19 12:31:36 +01001739 list_for_each_entry(obj_priv,
1740 &dev_priv->mm.inactive_list,
1741 list)
1742 {
1743 obj_priv->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1744 }
1745}
1746
Eric Anholt673a3942008-07-30 12:06:12 -07001747/**
1748 * This function clears the request list as sequence numbers are passed.
1749 */
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01001750static void
1751i915_gem_retire_requests_ring(struct drm_device *dev,
1752 struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001753{
1754 drm_i915_private_t *dev_priv = dev->dev_private;
1755 uint32_t seqno;
1756
Chris Wilsonb84d5f02010-09-18 01:38:04 +01001757 if (!ring->status_page.page_addr ||
1758 list_empty(&ring->request_list))
Karsten Wiese6c0594a2009-02-23 15:07:57 +01001759 return;
1760
Zou Nan hai852835f2010-05-21 09:08:56 +08001761 seqno = i915_get_gem_seqno(dev, ring);
Zou Nan hai852835f2010-05-21 09:08:56 +08001762 while (!list_empty(&ring->request_list)) {
Eric Anholt673a3942008-07-30 12:06:12 -07001763 struct drm_i915_gem_request *request;
Eric Anholt673a3942008-07-30 12:06:12 -07001764
Zou Nan hai852835f2010-05-21 09:08:56 +08001765 request = list_first_entry(&ring->request_list,
Eric Anholt673a3942008-07-30 12:06:12 -07001766 struct drm_i915_gem_request,
1767 list);
Eric Anholt673a3942008-07-30 12:06:12 -07001768
Chris Wilsondfaae392010-09-22 10:31:52 +01001769 if (!i915_seqno_passed(seqno, request->seqno))
Eric Anholt673a3942008-07-30 12:06:12 -07001770 break;
Chris Wilsonb84d5f02010-09-18 01:38:04 +01001771
1772 trace_i915_gem_request_retire(dev, request->seqno);
1773
1774 list_del(&request->list);
1775 list_del(&request->client_list);
1776 kfree(request);
1777 }
1778
1779 /* Move any buffers on the active list that are no longer referenced
1780 * by the ringbuffer to the flushing/inactive lists as appropriate.
1781 */
1782 while (!list_empty(&ring->active_list)) {
1783 struct drm_gem_object *obj;
1784 struct drm_i915_gem_object *obj_priv;
1785
1786 obj_priv = list_first_entry(&ring->active_list,
1787 struct drm_i915_gem_object,
1788 list);
1789
Chris Wilsondfaae392010-09-22 10:31:52 +01001790 if (!i915_seqno_passed(seqno, obj_priv->last_rendering_seqno))
Chris Wilsonb84d5f02010-09-18 01:38:04 +01001791 break;
1792
1793 obj = &obj_priv->base;
1794
1795#if WATCH_LRU
1796 DRM_INFO("%s: retire %d moves to inactive list %p\n",
1797 __func__, request->seqno, obj);
1798#endif
1799
1800 if (obj->write_domain != 0)
1801 i915_gem_object_move_to_flushing(obj);
1802 else
1803 i915_gem_object_move_to_inactive(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001804 }
Chris Wilson9d34e5d2009-09-24 05:26:06 +01001805
1806 if (unlikely (dev_priv->trace_irq_seqno &&
1807 i915_seqno_passed(dev_priv->trace_irq_seqno, seqno))) {
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001808 ring->user_irq_put(dev, ring);
Chris Wilson9d34e5d2009-09-24 05:26:06 +01001809 dev_priv->trace_irq_seqno = 0;
1810 }
Eric Anholt673a3942008-07-30 12:06:12 -07001811}
1812
1813void
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01001814i915_gem_retire_requests(struct drm_device *dev)
1815{
1816 drm_i915_private_t *dev_priv = dev->dev_private;
1817
Chris Wilsonbe726152010-07-23 23:18:50 +01001818 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1819 struct drm_i915_gem_object *obj_priv, *tmp;
1820
1821 /* We must be careful that during unbind() we do not
1822 * accidentally infinitely recurse into retire requests.
1823 * Currently:
1824 * retire -> free -> unbind -> wait -> retire_ring
1825 */
1826 list_for_each_entry_safe(obj_priv, tmp,
1827 &dev_priv->mm.deferred_free_list,
1828 list)
1829 i915_gem_free_object_tail(&obj_priv->base);
1830 }
1831
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01001832 i915_gem_retire_requests_ring(dev, &dev_priv->render_ring);
1833 if (HAS_BSD(dev))
1834 i915_gem_retire_requests_ring(dev, &dev_priv->bsd_ring);
1835}
1836
Daniel Vetter75ef9da2010-08-21 00:25:16 +02001837static void
Eric Anholt673a3942008-07-30 12:06:12 -07001838i915_gem_retire_work_handler(struct work_struct *work)
1839{
1840 drm_i915_private_t *dev_priv;
1841 struct drm_device *dev;
1842
1843 dev_priv = container_of(work, drm_i915_private_t,
1844 mm.retire_work.work);
1845 dev = dev_priv->dev;
1846
1847 mutex_lock(&dev->struct_mutex);
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01001848 i915_gem_retire_requests(dev);
Zou Nan haid1b851f2010-05-21 09:08:57 +08001849
Keith Packard6dbe2772008-10-14 21:41:13 -07001850 if (!dev_priv->mm.suspended &&
Zou Nan haid1b851f2010-05-21 09:08:57 +08001851 (!list_empty(&dev_priv->render_ring.request_list) ||
1852 (HAS_BSD(dev) &&
1853 !list_empty(&dev_priv->bsd_ring.request_list))))
Eric Anholt9c9fe1f2009-08-03 16:09:16 -07001854 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
Eric Anholt673a3942008-07-30 12:06:12 -07001855 mutex_unlock(&dev->struct_mutex);
1856}
1857
Daniel Vetter5a5a0c62009-09-15 22:57:36 +02001858int
Zou Nan hai852835f2010-05-21 09:08:56 +08001859i915_do_wait_request(struct drm_device *dev, uint32_t seqno,
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01001860 bool interruptible, struct intel_ring_buffer *ring)
Eric Anholt673a3942008-07-30 12:06:12 -07001861{
1862 drm_i915_private_t *dev_priv = dev->dev_private;
Jesse Barnes802c7eb2009-05-05 16:03:48 -07001863 u32 ier;
Eric Anholt673a3942008-07-30 12:06:12 -07001864 int ret = 0;
1865
1866 BUG_ON(seqno == 0);
1867
Daniel Vettere35a41d2010-02-11 22:13:59 +01001868 if (seqno == dev_priv->next_seqno) {
Chris Wilson8dc5d142010-08-12 12:36:12 +01001869 seqno = i915_add_request(dev, NULL, NULL, ring);
Daniel Vettere35a41d2010-02-11 22:13:59 +01001870 if (seqno == 0)
1871 return -ENOMEM;
1872 }
1873
Ben Gamariba1234d2009-09-14 17:48:47 -04001874 if (atomic_read(&dev_priv->mm.wedged))
Ben Gamariffed1d02009-09-14 17:48:41 -04001875 return -EIO;
1876
Zou Nan hai852835f2010-05-21 09:08:56 +08001877 if (!i915_seqno_passed(ring->get_gem_seqno(dev, ring), seqno)) {
Eric Anholtbad720f2009-10-22 16:11:14 -07001878 if (HAS_PCH_SPLIT(dev))
Zhenyu Wang036a4a72009-06-08 14:40:19 +08001879 ier = I915_READ(DEIER) | I915_READ(GTIER);
1880 else
1881 ier = I915_READ(IER);
Jesse Barnes802c7eb2009-05-05 16:03:48 -07001882 if (!ier) {
1883 DRM_ERROR("something (likely vbetool) disabled "
1884 "interrupts, re-enabling\n");
1885 i915_driver_irq_preinstall(dev);
1886 i915_driver_irq_postinstall(dev);
1887 }
1888
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001889 trace_i915_gem_request_wait_begin(dev, seqno);
1890
Zou Nan hai852835f2010-05-21 09:08:56 +08001891 ring->waiting_gem_seqno = seqno;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001892 ring->user_irq_get(dev, ring);
Daniel Vetter48764bf2009-09-15 22:57:32 +02001893 if (interruptible)
Zou Nan hai852835f2010-05-21 09:08:56 +08001894 ret = wait_event_interruptible(ring->irq_queue,
1895 i915_seqno_passed(
1896 ring->get_gem_seqno(dev, ring), seqno)
1897 || atomic_read(&dev_priv->mm.wedged));
Daniel Vetter48764bf2009-09-15 22:57:32 +02001898 else
Zou Nan hai852835f2010-05-21 09:08:56 +08001899 wait_event(ring->irq_queue,
1900 i915_seqno_passed(
1901 ring->get_gem_seqno(dev, ring), seqno)
1902 || atomic_read(&dev_priv->mm.wedged));
Daniel Vetter48764bf2009-09-15 22:57:32 +02001903
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001904 ring->user_irq_put(dev, ring);
Zou Nan hai852835f2010-05-21 09:08:56 +08001905 ring->waiting_gem_seqno = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01001906
1907 trace_i915_gem_request_wait_end(dev, seqno);
Eric Anholt673a3942008-07-30 12:06:12 -07001908 }
Ben Gamariba1234d2009-09-14 17:48:47 -04001909 if (atomic_read(&dev_priv->mm.wedged))
Eric Anholt673a3942008-07-30 12:06:12 -07001910 ret = -EIO;
1911
1912 if (ret && ret != -ERESTARTSYS)
Daniel Vetter8bff9172010-02-11 22:19:40 +01001913 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
1914 __func__, ret, seqno, ring->get_gem_seqno(dev, ring),
1915 dev_priv->next_seqno);
Eric Anholt673a3942008-07-30 12:06:12 -07001916
1917 /* Directly dispatch request retiring. While we have the work queue
1918 * to handle this, the waiter on a request often wants an associated
1919 * buffer to have made it to the inactive list, and we would need
1920 * a separate wait queue to handle that.
1921 */
1922 if (ret == 0)
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01001923 i915_gem_retire_requests_ring(dev, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07001924
1925 return ret;
1926}
1927
Daniel Vetter48764bf2009-09-15 22:57:32 +02001928/**
1929 * Waits for a sequence number to be signaled, and cleans up the
1930 * request and object lists appropriately for that event.
1931 */
1932static int
Zou Nan hai852835f2010-05-21 09:08:56 +08001933i915_wait_request(struct drm_device *dev, uint32_t seqno,
1934 struct intel_ring_buffer *ring)
Daniel Vetter48764bf2009-09-15 22:57:32 +02001935{
Zou Nan hai852835f2010-05-21 09:08:56 +08001936 return i915_do_wait_request(dev, seqno, 1, ring);
Daniel Vetter48764bf2009-09-15 22:57:32 +02001937}
1938
Chris Wilsonc7f9f9a2010-09-19 15:05:13 +01001939void
Chris Wilson92204342010-09-18 11:02:01 +01001940i915_gem_flush_ring(struct drm_device *dev,
Chris Wilsonc78ec302010-09-20 12:50:23 +01001941 struct drm_file *file_priv,
Chris Wilson92204342010-09-18 11:02:01 +01001942 struct intel_ring_buffer *ring,
1943 uint32_t invalidate_domains,
1944 uint32_t flush_domains)
1945{
1946 ring->flush(dev, ring, invalidate_domains, flush_domains);
1947 i915_gem_process_flushing_list(dev, flush_domains, ring);
Chris Wilsonc78ec302010-09-20 12:50:23 +01001948
1949 if (ring->outstanding_lazy_request) {
1950 (void)i915_add_request(dev, file_priv, NULL, ring);
1951 ring->outstanding_lazy_request = false;
1952 }
Chris Wilson92204342010-09-18 11:02:01 +01001953}
1954
1955static void
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001956i915_gem_flush(struct drm_device *dev,
Chris Wilsonc78ec302010-09-20 12:50:23 +01001957 struct drm_file *file_priv,
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001958 uint32_t invalidate_domains,
Chris Wilson92204342010-09-18 11:02:01 +01001959 uint32_t flush_domains,
1960 uint32_t flush_rings)
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001961{
1962 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter8bff9172010-02-11 22:19:40 +01001963
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001964 if (flush_domains & I915_GEM_DOMAIN_CPU)
1965 drm_agp_chipset_flush(dev);
Daniel Vetter8bff9172010-02-11 22:19:40 +01001966
Chris Wilson92204342010-09-18 11:02:01 +01001967 if ((flush_domains | invalidate_domains) & I915_GEM_GPU_DOMAINS) {
1968 if (flush_rings & RING_RENDER)
Chris Wilsonc78ec302010-09-20 12:50:23 +01001969 i915_gem_flush_ring(dev, file_priv,
Chris Wilson92204342010-09-18 11:02:01 +01001970 &dev_priv->render_ring,
1971 invalidate_domains, flush_domains);
1972 if (flush_rings & RING_BSD)
Chris Wilsonc78ec302010-09-20 12:50:23 +01001973 i915_gem_flush_ring(dev, file_priv,
Chris Wilson92204342010-09-18 11:02:01 +01001974 &dev_priv->bsd_ring,
1975 invalidate_domains, flush_domains);
1976 }
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001977}
1978
Eric Anholt673a3942008-07-30 12:06:12 -07001979/**
1980 * Ensures that all rendering to the object has completed and the object is
1981 * safe to unbind from the GTT or access from the CPU.
1982 */
1983static int
Chris Wilson2cf34d72010-09-14 13:03:28 +01001984i915_gem_object_wait_rendering(struct drm_gem_object *obj,
1985 bool interruptible)
Eric Anholt673a3942008-07-30 12:06:12 -07001986{
1987 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01001988 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001989 int ret;
1990
Eric Anholte47c68e2008-11-14 13:35:19 -08001991 /* This function only exists to support waiting for existing rendering,
1992 * not for emitting required flushes.
Eric Anholt673a3942008-07-30 12:06:12 -07001993 */
Eric Anholte47c68e2008-11-14 13:35:19 -08001994 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
Eric Anholt673a3942008-07-30 12:06:12 -07001995
1996 /* If there is rendering queued on the buffer being evicted, wait for
1997 * it.
1998 */
1999 if (obj_priv->active) {
2000#if WATCH_BUF
2001 DRM_INFO("%s: object %p wait for seqno %08x\n",
2002 __func__, obj, obj_priv->last_rendering_seqno);
2003#endif
Chris Wilson2cf34d72010-09-14 13:03:28 +01002004 ret = i915_do_wait_request(dev,
2005 obj_priv->last_rendering_seqno,
2006 interruptible,
2007 obj_priv->ring);
2008 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07002009 return ret;
2010 }
2011
2012 return 0;
2013}
2014
2015/**
2016 * Unbinds an object from the GTT aperture.
2017 */
Jesse Barnes0f973f22009-01-26 17:10:45 -08002018int
Eric Anholt673a3942008-07-30 12:06:12 -07002019i915_gem_object_unbind(struct drm_gem_object *obj)
2020{
2021 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01002022 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002023 int ret = 0;
2024
2025#if WATCH_BUF
2026 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
2027 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
2028#endif
2029 if (obj_priv->gtt_space == NULL)
2030 return 0;
2031
2032 if (obj_priv->pin_count != 0) {
2033 DRM_ERROR("Attempting to unbind pinned buffer\n");
2034 return -EINVAL;
2035 }
2036
Eric Anholt5323fd02009-09-09 11:50:45 -07002037 /* blow away mappings if mapped through GTT */
2038 i915_gem_release_mmap(obj);
2039
Eric Anholt673a3942008-07-30 12:06:12 -07002040 /* Move the object to the CPU domain to ensure that
2041 * any possible CPU writes while it's not in the GTT
2042 * are flushed when we go to remap it. This will
2043 * also ensure that all pending GPU writes are finished
2044 * before we unbind.
2045 */
Eric Anholte47c68e2008-11-14 13:35:19 -08002046 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
Chris Wilson8dc17752010-07-23 23:18:51 +01002047 if (ret == -ERESTARTSYS)
Eric Anholt673a3942008-07-30 12:06:12 -07002048 return ret;
Chris Wilson8dc17752010-07-23 23:18:51 +01002049 /* Continue on if we fail due to EIO, the GPU is hung so we
2050 * should be safe and we need to cleanup or else we might
2051 * cause memory corruption through use-after-free.
2052 */
Eric Anholt673a3942008-07-30 12:06:12 -07002053
Daniel Vetter96b47b62009-12-15 17:50:00 +01002054 /* release the fence reg _after_ flushing */
2055 if (obj_priv->fence_reg != I915_FENCE_REG_NONE)
2056 i915_gem_clear_fence_reg(obj);
2057
Eric Anholt673a3942008-07-30 12:06:12 -07002058 if (obj_priv->agp_mem != NULL) {
2059 drm_unbind_agp(obj_priv->agp_mem);
2060 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
2061 obj_priv->agp_mem = NULL;
2062 }
2063
Eric Anholt856fa192009-03-19 14:10:50 -07002064 i915_gem_object_put_pages(obj);
Chris Wilsona32808c2009-09-20 21:29:47 +01002065 BUG_ON(obj_priv->pages_refcount);
Eric Anholt673a3942008-07-30 12:06:12 -07002066
2067 if (obj_priv->gtt_space) {
2068 atomic_dec(&dev->gtt_count);
2069 atomic_sub(obj->size, &dev->gtt_memory);
2070
2071 drm_mm_put_block(obj_priv->gtt_space);
2072 obj_priv->gtt_space = NULL;
2073 }
2074
Chris Wilsonf13d3f72010-09-20 17:36:15 +01002075 list_del_init(&obj_priv->list);
Eric Anholt673a3942008-07-30 12:06:12 -07002076
Chris Wilson963b4832009-09-20 23:03:54 +01002077 if (i915_gem_object_is_purgeable(obj_priv))
2078 i915_gem_object_truncate(obj);
2079
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002080 trace_i915_gem_object_unbind(obj);
2081
Chris Wilson8dc17752010-07-23 23:18:51 +01002082 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002083}
2084
Chris Wilsonb47eb4a2010-08-07 11:01:23 +01002085int
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002086i915_gpu_idle(struct drm_device *dev)
2087{
2088 drm_i915_private_t *dev_priv = dev->dev_private;
2089 bool lists_empty;
Chris Wilsonc78ec302010-09-20 12:50:23 +01002090 u32 seqno;
Zou Nan hai852835f2010-05-21 09:08:56 +08002091 int ret;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002092
Zou Nan haid1b851f2010-05-21 09:08:57 +08002093 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2094 list_empty(&dev_priv->render_ring.active_list) &&
2095 (!HAS_BSD(dev) ||
2096 list_empty(&dev_priv->bsd_ring.active_list)));
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002097 if (lists_empty)
2098 return 0;
2099
2100 /* Flush everything onto the inactive list. */
Chris Wilsonc78ec302010-09-20 12:50:23 +01002101 seqno = i915_gem_next_request_seqno(dev, &dev_priv->render_ring);
2102 i915_gem_flush_ring(dev, NULL, &dev_priv->render_ring,
Chris Wilson92204342010-09-18 11:02:01 +01002103 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
Chris Wilsonc78ec302010-09-20 12:50:23 +01002104 ret = i915_wait_request(dev, seqno, &dev_priv->render_ring);
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01002105 if (ret)
2106 return ret;
Zou Nan haid1b851f2010-05-21 09:08:57 +08002107
2108 if (HAS_BSD(dev)) {
Chris Wilsonc78ec302010-09-20 12:50:23 +01002109 seqno = i915_gem_next_request_seqno(dev, &dev_priv->render_ring);
2110 i915_gem_flush_ring(dev, NULL, &dev_priv->bsd_ring,
Chris Wilson92204342010-09-18 11:02:01 +01002111 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
Chris Wilsonc78ec302010-09-20 12:50:23 +01002112 ret = i915_wait_request(dev, seqno, &dev_priv->bsd_ring);
Zou Nan haid1b851f2010-05-21 09:08:57 +08002113 if (ret)
2114 return ret;
2115 }
2116
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01002117 return 0;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01002118}
2119
Ben Gamari6911a9b2009-04-02 11:24:54 -07002120int
Chris Wilson4bdadb92010-01-27 13:36:32 +00002121i915_gem_object_get_pages(struct drm_gem_object *obj,
2122 gfp_t gfpmask)
Eric Anholt673a3942008-07-30 12:06:12 -07002123{
Daniel Vetter23010e42010-03-08 13:35:02 +01002124 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002125 int page_count, i;
2126 struct address_space *mapping;
2127 struct inode *inode;
2128 struct page *page;
Eric Anholt673a3942008-07-30 12:06:12 -07002129
Daniel Vetter778c3542010-05-13 11:49:44 +02002130 BUG_ON(obj_priv->pages_refcount
2131 == DRM_I915_GEM_OBJECT_MAX_PAGES_REFCOUNT);
2132
Eric Anholt856fa192009-03-19 14:10:50 -07002133 if (obj_priv->pages_refcount++ != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07002134 return 0;
2135
2136 /* Get the list of pages out of our struct file. They'll be pinned
2137 * at this point until we release them.
2138 */
2139 page_count = obj->size / PAGE_SIZE;
Eric Anholt856fa192009-03-19 14:10:50 -07002140 BUG_ON(obj_priv->pages != NULL);
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07002141 obj_priv->pages = drm_calloc_large(page_count, sizeof(struct page *));
Eric Anholt856fa192009-03-19 14:10:50 -07002142 if (obj_priv->pages == NULL) {
Eric Anholt856fa192009-03-19 14:10:50 -07002143 obj_priv->pages_refcount--;
Eric Anholt673a3942008-07-30 12:06:12 -07002144 return -ENOMEM;
2145 }
2146
2147 inode = obj->filp->f_path.dentry->d_inode;
2148 mapping = inode->i_mapping;
2149 for (i = 0; i < page_count; i++) {
Chris Wilson4bdadb92010-01-27 13:36:32 +00002150 page = read_cache_page_gfp(mapping, i,
Linus Torvalds985b8232010-07-02 10:04:42 +10002151 GFP_HIGHUSER |
Chris Wilson4bdadb92010-01-27 13:36:32 +00002152 __GFP_COLD |
Linus Torvaldscd9f0402010-07-18 09:44:37 -07002153 __GFP_RECLAIMABLE |
Chris Wilson4bdadb92010-01-27 13:36:32 +00002154 gfpmask);
Chris Wilson1f2b1012010-03-12 19:52:55 +00002155 if (IS_ERR(page))
2156 goto err_pages;
2157
Eric Anholt856fa192009-03-19 14:10:50 -07002158 obj_priv->pages[i] = page;
Eric Anholt673a3942008-07-30 12:06:12 -07002159 }
Eric Anholt280b7132009-03-12 16:56:27 -07002160
2161 if (obj_priv->tiling_mode != I915_TILING_NONE)
2162 i915_gem_object_do_bit_17_swizzle(obj);
2163
Eric Anholt673a3942008-07-30 12:06:12 -07002164 return 0;
Chris Wilson1f2b1012010-03-12 19:52:55 +00002165
2166err_pages:
2167 while (i--)
2168 page_cache_release(obj_priv->pages[i]);
2169
2170 drm_free_large(obj_priv->pages);
2171 obj_priv->pages = NULL;
2172 obj_priv->pages_refcount--;
2173 return PTR_ERR(page);
Eric Anholt673a3942008-07-30 12:06:12 -07002174}
2175
Eric Anholt4e901fd2009-10-26 16:44:17 -07002176static void sandybridge_write_fence_reg(struct drm_i915_fence_reg *reg)
2177{
2178 struct drm_gem_object *obj = reg->obj;
2179 struct drm_device *dev = obj->dev;
2180 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002181 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt4e901fd2009-10-26 16:44:17 -07002182 int regnum = obj_priv->fence_reg;
2183 uint64_t val;
2184
2185 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2186 0xfffff000) << 32;
2187 val |= obj_priv->gtt_offset & 0xfffff000;
2188 val |= (uint64_t)((obj_priv->stride / 128) - 1) <<
2189 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2190
2191 if (obj_priv->tiling_mode == I915_TILING_Y)
2192 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2193 val |= I965_FENCE_REG_VALID;
2194
2195 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (regnum * 8), val);
2196}
2197
Jesse Barnesde151cf2008-11-12 10:03:55 -08002198static void i965_write_fence_reg(struct drm_i915_fence_reg *reg)
2199{
2200 struct drm_gem_object *obj = reg->obj;
2201 struct drm_device *dev = obj->dev;
2202 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002203 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002204 int regnum = obj_priv->fence_reg;
2205 uint64_t val;
2206
2207 val = (uint64_t)((obj_priv->gtt_offset + obj->size - 4096) &
2208 0xfffff000) << 32;
2209 val |= obj_priv->gtt_offset & 0xfffff000;
2210 val |= ((obj_priv->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2211 if (obj_priv->tiling_mode == I915_TILING_Y)
2212 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2213 val |= I965_FENCE_REG_VALID;
2214
2215 I915_WRITE64(FENCE_REG_965_0 + (regnum * 8), val);
2216}
2217
2218static void i915_write_fence_reg(struct drm_i915_fence_reg *reg)
2219{
2220 struct drm_gem_object *obj = reg->obj;
2221 struct drm_device *dev = obj->dev;
2222 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002223 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002224 int regnum = obj_priv->fence_reg;
Jesse Barnes0f973f22009-01-26 17:10:45 -08002225 int tile_width;
Eric Anholtdc529a42009-03-10 22:34:49 -07002226 uint32_t fence_reg, val;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002227 uint32_t pitch_val;
2228
2229 if ((obj_priv->gtt_offset & ~I915_FENCE_START_MASK) ||
2230 (obj_priv->gtt_offset & (obj->size - 1))) {
Linus Torvaldsf06da262009-02-09 08:57:29 -08002231 WARN(1, "%s: object 0x%08x not 1M or size (0x%zx) aligned\n",
Jesse Barnes0f973f22009-01-26 17:10:45 -08002232 __func__, obj_priv->gtt_offset, obj->size);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002233 return;
2234 }
2235
Jesse Barnes0f973f22009-01-26 17:10:45 -08002236 if (obj_priv->tiling_mode == I915_TILING_Y &&
2237 HAS_128_BYTE_Y_TILING(dev))
2238 tile_width = 128;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002239 else
Jesse Barnes0f973f22009-01-26 17:10:45 -08002240 tile_width = 512;
2241
2242 /* Note: pitch better be a power of two tile widths */
2243 pitch_val = obj_priv->stride / tile_width;
2244 pitch_val = ffs(pitch_val) - 1;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002245
Daniel Vetterc36a2a62010-04-17 15:12:03 +02002246 if (obj_priv->tiling_mode == I915_TILING_Y &&
2247 HAS_128_BYTE_Y_TILING(dev))
2248 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2249 else
2250 WARN_ON(pitch_val > I915_FENCE_MAX_PITCH_VAL);
2251
Jesse Barnesde151cf2008-11-12 10:03:55 -08002252 val = obj_priv->gtt_offset;
2253 if (obj_priv->tiling_mode == I915_TILING_Y)
2254 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2255 val |= I915_FENCE_SIZE_BITS(obj->size);
2256 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2257 val |= I830_FENCE_REG_VALID;
2258
Eric Anholtdc529a42009-03-10 22:34:49 -07002259 if (regnum < 8)
2260 fence_reg = FENCE_REG_830_0 + (regnum * 4);
2261 else
2262 fence_reg = FENCE_REG_945_8 + ((regnum - 8) * 4);
2263 I915_WRITE(fence_reg, val);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002264}
2265
2266static void i830_write_fence_reg(struct drm_i915_fence_reg *reg)
2267{
2268 struct drm_gem_object *obj = reg->obj;
2269 struct drm_device *dev = obj->dev;
2270 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002271 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002272 int regnum = obj_priv->fence_reg;
2273 uint32_t val;
2274 uint32_t pitch_val;
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002275 uint32_t fence_size_bits;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002276
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002277 if ((obj_priv->gtt_offset & ~I830_FENCE_START_MASK) ||
Jesse Barnesde151cf2008-11-12 10:03:55 -08002278 (obj_priv->gtt_offset & (obj->size - 1))) {
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002279 WARN(1, "%s: object 0x%08x not 512K or size aligned\n",
Jesse Barnes0f973f22009-01-26 17:10:45 -08002280 __func__, obj_priv->gtt_offset);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002281 return;
2282 }
2283
Eric Anholte76a16d2009-05-26 17:44:56 -07002284 pitch_val = obj_priv->stride / 128;
2285 pitch_val = ffs(pitch_val) - 1;
2286 WARN_ON(pitch_val > I830_FENCE_MAX_PITCH_VAL);
2287
Jesse Barnesde151cf2008-11-12 10:03:55 -08002288 val = obj_priv->gtt_offset;
2289 if (obj_priv->tiling_mode == I915_TILING_Y)
2290 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002291 fence_size_bits = I830_FENCE_SIZE_BITS(obj->size);
2292 WARN_ON(fence_size_bits & ~0x00000f00);
2293 val |= fence_size_bits;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002294 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2295 val |= I830_FENCE_REG_VALID;
2296
2297 I915_WRITE(FENCE_REG_830_0 + (regnum * 4), val);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002298}
2299
Chris Wilson2cf34d72010-09-14 13:03:28 +01002300static int i915_find_fence_reg(struct drm_device *dev,
2301 bool interruptible)
Daniel Vetterae3db242010-02-19 11:51:58 +01002302{
2303 struct drm_i915_fence_reg *reg = NULL;
2304 struct drm_i915_gem_object *obj_priv = NULL;
2305 struct drm_i915_private *dev_priv = dev->dev_private;
2306 struct drm_gem_object *obj = NULL;
2307 int i, avail, ret;
2308
2309 /* First try to find a free reg */
2310 avail = 0;
2311 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2312 reg = &dev_priv->fence_regs[i];
2313 if (!reg->obj)
2314 return i;
2315
Daniel Vetter23010e42010-03-08 13:35:02 +01002316 obj_priv = to_intel_bo(reg->obj);
Daniel Vetterae3db242010-02-19 11:51:58 +01002317 if (!obj_priv->pin_count)
2318 avail++;
2319 }
2320
2321 if (avail == 0)
2322 return -ENOSPC;
2323
2324 /* None available, try to steal one or wait for a user to finish */
2325 i = I915_FENCE_REG_NONE;
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002326 list_for_each_entry(reg, &dev_priv->mm.fence_list,
2327 lru_list) {
2328 obj = reg->obj;
2329 obj_priv = to_intel_bo(obj);
Daniel Vetterae3db242010-02-19 11:51:58 +01002330
2331 if (obj_priv->pin_count)
2332 continue;
2333
2334 /* found one! */
2335 i = obj_priv->fence_reg;
2336 break;
2337 }
2338
2339 BUG_ON(i == I915_FENCE_REG_NONE);
2340
2341 /* We only have a reference on obj from the active list. put_fence_reg
2342 * might drop that one, causing a use-after-free in it. So hold a
2343 * private reference to obj like the other callers of put_fence_reg
2344 * (set_tiling ioctl) do. */
2345 drm_gem_object_reference(obj);
Chris Wilson2cf34d72010-09-14 13:03:28 +01002346 ret = i915_gem_object_put_fence_reg(obj, interruptible);
Daniel Vetterae3db242010-02-19 11:51:58 +01002347 drm_gem_object_unreference(obj);
2348 if (ret != 0)
2349 return ret;
2350
2351 return i;
2352}
2353
Jesse Barnesde151cf2008-11-12 10:03:55 -08002354/**
2355 * i915_gem_object_get_fence_reg - set up a fence reg for an object
2356 * @obj: object to map through a fence reg
2357 *
2358 * When mapping objects through the GTT, userspace wants to be able to write
2359 * to them without having to worry about swizzling if the object is tiled.
2360 *
2361 * This function walks the fence regs looking for a free one for @obj,
2362 * stealing one if it can't find any.
2363 *
2364 * It then sets up the reg based on the object's properties: address, pitch
2365 * and tiling format.
2366 */
Chris Wilson8c4b8c32009-06-17 22:08:52 +01002367int
Chris Wilson2cf34d72010-09-14 13:03:28 +01002368i915_gem_object_get_fence_reg(struct drm_gem_object *obj,
2369 bool interruptible)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002370{
2371 struct drm_device *dev = obj->dev;
Jesse Barnes79e53942008-11-07 14:24:08 -08002372 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002373 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002374 struct drm_i915_fence_reg *reg = NULL;
Daniel Vetterae3db242010-02-19 11:51:58 +01002375 int ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002376
Eric Anholta09ba7f2009-08-29 12:49:51 -07002377 /* Just update our place in the LRU if our fence is getting used. */
2378 if (obj_priv->fence_reg != I915_FENCE_REG_NONE) {
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002379 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2380 list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07002381 return 0;
2382 }
2383
Jesse Barnesde151cf2008-11-12 10:03:55 -08002384 switch (obj_priv->tiling_mode) {
2385 case I915_TILING_NONE:
2386 WARN(1, "allocating a fence for non-tiled object?\n");
2387 break;
2388 case I915_TILING_X:
Jesse Barnes0f973f22009-01-26 17:10:45 -08002389 if (!obj_priv->stride)
2390 return -EINVAL;
2391 WARN((obj_priv->stride & (512 - 1)),
2392 "object 0x%08x is X tiled but has non-512B pitch\n",
2393 obj_priv->gtt_offset);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002394 break;
2395 case I915_TILING_Y:
Jesse Barnes0f973f22009-01-26 17:10:45 -08002396 if (!obj_priv->stride)
2397 return -EINVAL;
2398 WARN((obj_priv->stride & (128 - 1)),
2399 "object 0x%08x is Y tiled but has non-128B pitch\n",
2400 obj_priv->gtt_offset);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002401 break;
2402 }
2403
Chris Wilson2cf34d72010-09-14 13:03:28 +01002404 ret = i915_find_fence_reg(dev, interruptible);
Daniel Vetterae3db242010-02-19 11:51:58 +01002405 if (ret < 0)
2406 return ret;
Chris Wilsonfc7170b2009-02-11 14:26:46 +00002407
Daniel Vetterae3db242010-02-19 11:51:58 +01002408 obj_priv->fence_reg = ret;
2409 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002410 list_add_tail(&reg->lru_list, &dev_priv->mm.fence_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07002411
Jesse Barnesde151cf2008-11-12 10:03:55 -08002412 reg->obj = obj;
2413
Chris Wilsone259bef2010-09-17 00:32:02 +01002414 switch (INTEL_INFO(dev)->gen) {
2415 case 6:
Eric Anholt4e901fd2009-10-26 16:44:17 -07002416 sandybridge_write_fence_reg(reg);
Chris Wilsone259bef2010-09-17 00:32:02 +01002417 break;
2418 case 5:
2419 case 4:
Jesse Barnesde151cf2008-11-12 10:03:55 -08002420 i965_write_fence_reg(reg);
Chris Wilsone259bef2010-09-17 00:32:02 +01002421 break;
2422 case 3:
Jesse Barnesde151cf2008-11-12 10:03:55 -08002423 i915_write_fence_reg(reg);
Chris Wilsone259bef2010-09-17 00:32:02 +01002424 break;
2425 case 2:
Jesse Barnesde151cf2008-11-12 10:03:55 -08002426 i830_write_fence_reg(reg);
Chris Wilsone259bef2010-09-17 00:32:02 +01002427 break;
2428 }
Eric Anholtd9ddcb92009-01-27 10:33:49 -08002429
Daniel Vetterae3db242010-02-19 11:51:58 +01002430 trace_i915_gem_object_get_fence(obj, obj_priv->fence_reg,
2431 obj_priv->tiling_mode);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002432
Eric Anholtd9ddcb92009-01-27 10:33:49 -08002433 return 0;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002434}
2435
2436/**
2437 * i915_gem_clear_fence_reg - clear out fence register info
2438 * @obj: object to clear
2439 *
2440 * Zeroes out the fence register itself and clears out the associated
2441 * data structures in dev_priv and obj_priv.
2442 */
2443static void
2444i915_gem_clear_fence_reg(struct drm_gem_object *obj)
2445{
2446 struct drm_device *dev = obj->dev;
Jesse Barnes79e53942008-11-07 14:24:08 -08002447 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002448 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002449 struct drm_i915_fence_reg *reg =
2450 &dev_priv->fence_regs[obj_priv->fence_reg];
Chris Wilsone259bef2010-09-17 00:32:02 +01002451 uint32_t fence_reg;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002452
Chris Wilsone259bef2010-09-17 00:32:02 +01002453 switch (INTEL_INFO(dev)->gen) {
2454 case 6:
Eric Anholt4e901fd2009-10-26 16:44:17 -07002455 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 +
2456 (obj_priv->fence_reg * 8), 0);
Chris Wilsone259bef2010-09-17 00:32:02 +01002457 break;
2458 case 5:
2459 case 4:
Jesse Barnesde151cf2008-11-12 10:03:55 -08002460 I915_WRITE64(FENCE_REG_965_0 + (obj_priv->fence_reg * 8), 0);
Chris Wilsone259bef2010-09-17 00:32:02 +01002461 break;
2462 case 3:
2463 if (obj_priv->fence_reg > 8)
2464 fence_reg = FENCE_REG_945_8 + (obj_priv->fence_reg - 8) * 4;
Eric Anholtdc529a42009-03-10 22:34:49 -07002465 else
Chris Wilsone259bef2010-09-17 00:32:02 +01002466 case 2:
2467 fence_reg = FENCE_REG_830_0 + obj_priv->fence_reg * 4;
Eric Anholtdc529a42009-03-10 22:34:49 -07002468
2469 I915_WRITE(fence_reg, 0);
Chris Wilsone259bef2010-09-17 00:32:02 +01002470 break;
Eric Anholtdc529a42009-03-10 22:34:49 -07002471 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08002472
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002473 reg->obj = NULL;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002474 obj_priv->fence_reg = I915_FENCE_REG_NONE;
Daniel Vetter007cc8a2010-04-28 11:02:31 +02002475 list_del_init(&reg->lru_list);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002476}
2477
Eric Anholt673a3942008-07-30 12:06:12 -07002478/**
Chris Wilson52dc7d32009-06-06 09:46:01 +01002479 * i915_gem_object_put_fence_reg - waits on outstanding fenced access
2480 * to the buffer to finish, and then resets the fence register.
2481 * @obj: tiled object holding a fence register.
Chris Wilson2cf34d72010-09-14 13:03:28 +01002482 * @bool: whether the wait upon the fence is interruptible
Chris Wilson52dc7d32009-06-06 09:46:01 +01002483 *
2484 * Zeroes out the fence register itself and clears out the associated
2485 * data structures in dev_priv and obj_priv.
2486 */
2487int
Chris Wilson2cf34d72010-09-14 13:03:28 +01002488i915_gem_object_put_fence_reg(struct drm_gem_object *obj,
2489 bool interruptible)
Chris Wilson52dc7d32009-06-06 09:46:01 +01002490{
2491 struct drm_device *dev = obj->dev;
Chris Wilson53640e12010-09-20 11:40:50 +01002492 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002493 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson53640e12010-09-20 11:40:50 +01002494 struct drm_i915_fence_reg *reg;
Chris Wilson52dc7d32009-06-06 09:46:01 +01002495
2496 if (obj_priv->fence_reg == I915_FENCE_REG_NONE)
2497 return 0;
2498
Daniel Vetter10ae9bd2010-02-01 13:59:17 +01002499 /* If we've changed tiling, GTT-mappings of the object
2500 * need to re-fault to ensure that the correct fence register
2501 * setup is in place.
2502 */
2503 i915_gem_release_mmap(obj);
2504
Chris Wilson52dc7d32009-06-06 09:46:01 +01002505 /* On the i915, GPU access to tiled buffers is via a fence,
2506 * therefore we must wait for any outstanding access to complete
2507 * before clearing the fence.
2508 */
Chris Wilson53640e12010-09-20 11:40:50 +01002509 reg = &dev_priv->fence_regs[obj_priv->fence_reg];
2510 if (reg->gpu) {
Chris Wilson52dc7d32009-06-06 09:46:01 +01002511 int ret;
2512
Chris Wilson2cf34d72010-09-14 13:03:28 +01002513 ret = i915_gem_object_flush_gpu_write_domain(obj, true);
Chris Wilson0bc23aa2010-09-14 10:22:23 +01002514 if (ret)
2515 return ret;
2516
Chris Wilson2cf34d72010-09-14 13:03:28 +01002517 ret = i915_gem_object_wait_rendering(obj, interruptible);
Chris Wilson0bc23aa2010-09-14 10:22:23 +01002518 if (ret)
Chris Wilson52dc7d32009-06-06 09:46:01 +01002519 return ret;
Chris Wilson53640e12010-09-20 11:40:50 +01002520
2521 reg->gpu = false;
Chris Wilson52dc7d32009-06-06 09:46:01 +01002522 }
2523
Daniel Vetter4a726612010-02-01 13:59:16 +01002524 i915_gem_object_flush_gtt_write_domain(obj);
Chris Wilson0bc23aa2010-09-14 10:22:23 +01002525 i915_gem_clear_fence_reg(obj);
Chris Wilson52dc7d32009-06-06 09:46:01 +01002526
2527 return 0;
2528}
2529
2530/**
Eric Anholt673a3942008-07-30 12:06:12 -07002531 * Finds free space in the GTT aperture and binds the object there.
2532 */
2533static int
2534i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
2535{
2536 struct drm_device *dev = obj->dev;
2537 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002538 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002539 struct drm_mm_node *free_space;
Chris Wilson4bdadb92010-01-27 13:36:32 +00002540 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
Chris Wilson07f73f62009-09-14 16:50:30 +01002541 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002542
Chris Wilsonbb6baf72009-09-22 14:24:13 +01002543 if (obj_priv->madv != I915_MADV_WILLNEED) {
Chris Wilson3ef94da2009-09-14 16:50:29 +01002544 DRM_ERROR("Attempting to bind a purgeable object\n");
2545 return -EINVAL;
2546 }
2547
Eric Anholt673a3942008-07-30 12:06:12 -07002548 if (alignment == 0)
Jesse Barnes0f973f22009-01-26 17:10:45 -08002549 alignment = i915_gem_get_gtt_alignment(obj);
Daniel Vetter8d7773a2009-03-29 14:09:41 +02002550 if (alignment & (i915_gem_get_gtt_alignment(obj) - 1)) {
Eric Anholt673a3942008-07-30 12:06:12 -07002551 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2552 return -EINVAL;
2553 }
2554
Chris Wilson654fc602010-05-27 13:18:21 +01002555 /* If the object is bigger than the entire aperture, reject it early
2556 * before evicting everything in a vain attempt to find space.
2557 */
2558 if (obj->size > dev->gtt_total) {
2559 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2560 return -E2BIG;
2561 }
2562
Eric Anholt673a3942008-07-30 12:06:12 -07002563 search_free:
2564 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2565 obj->size, alignment, 0);
2566 if (free_space != NULL) {
2567 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
2568 alignment);
Daniel Vetterdb3307a2010-07-02 15:02:12 +01002569 if (obj_priv->gtt_space != NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07002570 obj_priv->gtt_offset = obj_priv->gtt_space->start;
Eric Anholt673a3942008-07-30 12:06:12 -07002571 }
2572 if (obj_priv->gtt_space == NULL) {
2573 /* If the gtt is empty and we're still having trouble
2574 * fitting our object in, we're out of memory.
2575 */
2576#if WATCH_LRU
2577 DRM_INFO("%s: GTT full, evicting something\n", __func__);
2578#endif
Daniel Vetter0108a3e2010-08-07 11:01:21 +01002579 ret = i915_gem_evict_something(dev, obj->size, alignment);
Chris Wilson97311292009-09-21 00:22:34 +01002580 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07002581 return ret;
Chris Wilson97311292009-09-21 00:22:34 +01002582
Eric Anholt673a3942008-07-30 12:06:12 -07002583 goto search_free;
2584 }
2585
2586#if WATCH_BUF
Krzysztof Halasacfd43c02009-06-20 00:31:28 +02002587 DRM_INFO("Binding object of size %zd at 0x%08x\n",
Eric Anholt673a3942008-07-30 12:06:12 -07002588 obj->size, obj_priv->gtt_offset);
2589#endif
Chris Wilson4bdadb92010-01-27 13:36:32 +00002590 ret = i915_gem_object_get_pages(obj, gfpmask);
Eric Anholt673a3942008-07-30 12:06:12 -07002591 if (ret) {
2592 drm_mm_put_block(obj_priv->gtt_space);
2593 obj_priv->gtt_space = NULL;
Chris Wilson07f73f62009-09-14 16:50:30 +01002594
2595 if (ret == -ENOMEM) {
2596 /* first try to clear up some space from the GTT */
Daniel Vetter0108a3e2010-08-07 11:01:21 +01002597 ret = i915_gem_evict_something(dev, obj->size,
2598 alignment);
Chris Wilson07f73f62009-09-14 16:50:30 +01002599 if (ret) {
Chris Wilson07f73f62009-09-14 16:50:30 +01002600 /* now try to shrink everyone else */
Chris Wilson4bdadb92010-01-27 13:36:32 +00002601 if (gfpmask) {
2602 gfpmask = 0;
2603 goto search_free;
Chris Wilson07f73f62009-09-14 16:50:30 +01002604 }
2605
2606 return ret;
2607 }
2608
2609 goto search_free;
2610 }
2611
Eric Anholt673a3942008-07-30 12:06:12 -07002612 return ret;
2613 }
2614
Eric Anholt673a3942008-07-30 12:06:12 -07002615 /* Create an AGP memory structure pointing at our pages, and bind it
2616 * into the GTT.
2617 */
2618 obj_priv->agp_mem = drm_agp_bind_pages(dev,
Eric Anholt856fa192009-03-19 14:10:50 -07002619 obj_priv->pages,
Chris Wilson07f73f62009-09-14 16:50:30 +01002620 obj->size >> PAGE_SHIFT,
Keith Packardba1eb1d2008-10-14 19:55:10 -07002621 obj_priv->gtt_offset,
2622 obj_priv->agp_type);
Eric Anholt673a3942008-07-30 12:06:12 -07002623 if (obj_priv->agp_mem == NULL) {
Eric Anholt856fa192009-03-19 14:10:50 -07002624 i915_gem_object_put_pages(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002625 drm_mm_put_block(obj_priv->gtt_space);
2626 obj_priv->gtt_space = NULL;
Chris Wilson07f73f62009-09-14 16:50:30 +01002627
Daniel Vetter0108a3e2010-08-07 11:01:21 +01002628 ret = i915_gem_evict_something(dev, obj->size, alignment);
Chris Wilson97311292009-09-21 00:22:34 +01002629 if (ret)
Chris Wilson07f73f62009-09-14 16:50:30 +01002630 return ret;
Chris Wilson07f73f62009-09-14 16:50:30 +01002631
2632 goto search_free;
Eric Anholt673a3942008-07-30 12:06:12 -07002633 }
2634 atomic_inc(&dev->gtt_count);
2635 atomic_add(obj->size, &dev->gtt_memory);
2636
Chris Wilsonbf1a1092010-08-07 11:01:20 +01002637 /* keep track of bounds object by adding it to the inactive list */
2638 list_add_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
2639
Eric Anholt673a3942008-07-30 12:06:12 -07002640 /* Assert that the object is not currently in any GPU domain. As it
2641 * wasn't in the GTT, there shouldn't be any way it could have been in
2642 * a GPU cache
2643 */
Chris Wilson21d509e2009-06-06 09:46:02 +01002644 BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2645 BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
Eric Anholt673a3942008-07-30 12:06:12 -07002646
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002647 trace_i915_gem_object_bind(obj, obj_priv->gtt_offset);
2648
Eric Anholt673a3942008-07-30 12:06:12 -07002649 return 0;
2650}
2651
2652void
2653i915_gem_clflush_object(struct drm_gem_object *obj)
2654{
Daniel Vetter23010e42010-03-08 13:35:02 +01002655 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002656
2657 /* If we don't have a page list set up, then we're not pinned
2658 * to GPU, and we can ignore the cache flush because it'll happen
2659 * again at bind time.
2660 */
Eric Anholt856fa192009-03-19 14:10:50 -07002661 if (obj_priv->pages == NULL)
Eric Anholt673a3942008-07-30 12:06:12 -07002662 return;
2663
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002664 trace_i915_gem_object_clflush(obj);
Eric Anholtcfa16a02009-05-26 18:46:16 -07002665
Eric Anholt856fa192009-03-19 14:10:50 -07002666 drm_clflush_pages(obj_priv->pages, obj->size / PAGE_SIZE);
Eric Anholt673a3942008-07-30 12:06:12 -07002667}
2668
Eric Anholte47c68e2008-11-14 13:35:19 -08002669/** Flushes any GPU write domain for the object if it's dirty. */
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002670static int
Daniel Vetterba3d8d72010-02-11 22:37:04 +01002671i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj,
2672 bool pipelined)
Eric Anholte47c68e2008-11-14 13:35:19 -08002673{
2674 struct drm_device *dev = obj->dev;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002675 uint32_t old_write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002676
2677 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
Chris Wilson2dafb1e2010-06-07 14:03:05 +01002678 return 0;
Eric Anholte47c68e2008-11-14 13:35:19 -08002679
2680 /* Queue the GPU write cache flushing we need. */
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002681 old_write_domain = obj->write_domain;
Chris Wilsonc78ec302010-09-20 12:50:23 +01002682 i915_gem_flush_ring(dev, NULL,
Chris Wilson92204342010-09-18 11:02:01 +01002683 to_intel_bo(obj)->ring,
2684 0, obj->write_domain);
Chris Wilson48b956c2010-09-14 12:50:34 +01002685 BUG_ON(obj->write_domain);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002686
2687 trace_i915_gem_object_change_domain(obj,
2688 obj->read_domains,
2689 old_write_domain);
Daniel Vetterba3d8d72010-02-11 22:37:04 +01002690
2691 if (pipelined)
2692 return 0;
2693
Chris Wilson2cf34d72010-09-14 13:03:28 +01002694 return i915_gem_object_wait_rendering(obj, true);
Eric Anholte47c68e2008-11-14 13:35:19 -08002695}
2696
2697/** Flushes the GTT write domain for the object if it's dirty. */
2698static void
2699i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
2700{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002701 uint32_t old_write_domain;
2702
Eric Anholte47c68e2008-11-14 13:35:19 -08002703 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
2704 return;
2705
2706 /* No actual flushing is required for the GTT write domain. Writes
2707 * to it immediately go to main memory as far as we know, so there's
2708 * no chipset flush. It also doesn't land in render cache.
2709 */
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002710 old_write_domain = obj->write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002711 obj->write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002712
2713 trace_i915_gem_object_change_domain(obj,
2714 obj->read_domains,
2715 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08002716}
2717
2718/** Flushes the CPU write domain for the object if it's dirty. */
2719static void
2720i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
2721{
2722 struct drm_device *dev = obj->dev;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002723 uint32_t old_write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002724
2725 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
2726 return;
2727
2728 i915_gem_clflush_object(obj);
2729 drm_agp_chipset_flush(dev);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002730 old_write_domain = obj->write_domain;
Eric Anholte47c68e2008-11-14 13:35:19 -08002731 obj->write_domain = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002732
2733 trace_i915_gem_object_change_domain(obj,
2734 obj->read_domains,
2735 old_write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -08002736}
2737
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002738/**
2739 * Moves a single object to the GTT read, and possibly write domain.
2740 *
2741 * This function returns when the move is complete, including waiting on
2742 * flushes to occur.
2743 */
Jesse Barnes79e53942008-11-07 14:24:08 -08002744int
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002745i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
2746{
Daniel Vetter23010e42010-03-08 13:35:02 +01002747 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002748 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08002749 int ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002750
Eric Anholt02354392008-11-26 13:58:13 -08002751 /* Not valid to be called on unbound objects. */
2752 if (obj_priv->gtt_space == NULL)
2753 return -EINVAL;
2754
Daniel Vetterba3d8d72010-02-11 22:37:04 +01002755 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
Eric Anholte47c68e2008-11-14 13:35:19 -08002756 if (ret != 0)
2757 return ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002758
Chris Wilson72133422010-09-13 23:56:38 +01002759 i915_gem_object_flush_cpu_write_domain(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002760
Daniel Vetterba3d8d72010-02-11 22:37:04 +01002761 if (write) {
Chris Wilson2cf34d72010-09-14 13:03:28 +01002762 ret = i915_gem_object_wait_rendering(obj, true);
Daniel Vetterba3d8d72010-02-11 22:37:04 +01002763 if (ret)
2764 return ret;
Daniel Vetterba3d8d72010-02-11 22:37:04 +01002765 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002766
Chris Wilson72133422010-09-13 23:56:38 +01002767 old_write_domain = obj->write_domain;
2768 old_read_domains = obj->read_domains;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002769
2770 /* It should now be out of any other write domains, and we can update
2771 * the domain values for our changes.
2772 */
2773 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
2774 obj->read_domains |= I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08002775 if (write) {
Chris Wilson72133422010-09-13 23:56:38 +01002776 obj->read_domains = I915_GEM_DOMAIN_GTT;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002777 obj->write_domain = I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08002778 obj_priv->dirty = 1;
2779 }
2780
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002781 trace_i915_gem_object_change_domain(obj,
2782 old_read_domains,
2783 old_write_domain);
2784
Eric Anholte47c68e2008-11-14 13:35:19 -08002785 return 0;
2786}
2787
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002788/*
2789 * Prepare buffer for display plane. Use uninterruptible for possible flush
2790 * wait, as in modesetting process we're not supposed to be interrupted.
2791 */
2792int
Chris Wilson48b956c2010-09-14 12:50:34 +01002793i915_gem_object_set_to_display_plane(struct drm_gem_object *obj,
2794 bool pipelined)
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002795{
Daniel Vetter23010e42010-03-08 13:35:02 +01002796 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Daniel Vetterba3d8d72010-02-11 22:37:04 +01002797 uint32_t old_read_domains;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002798 int ret;
2799
2800 /* Not valid to be called on unbound objects. */
2801 if (obj_priv->gtt_space == NULL)
2802 return -EINVAL;
2803
Chris Wilson48b956c2010-09-14 12:50:34 +01002804 ret = i915_gem_object_flush_gpu_write_domain(obj, pipelined);
2805 if (ret)
Daniel Vettere35a41d2010-02-11 22:13:59 +01002806 return ret;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002807
Chris Wilsonb118c1e2010-05-27 13:18:14 +01002808 i915_gem_object_flush_cpu_write_domain(obj);
2809
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002810 old_read_domains = obj->read_domains;
Chris Wilsonc78ec302010-09-20 12:50:23 +01002811 obj->read_domains |= I915_GEM_DOMAIN_GTT;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002812
2813 trace_i915_gem_object_change_domain(obj,
2814 old_read_domains,
Daniel Vetterba3d8d72010-02-11 22:37:04 +01002815 obj->write_domain);
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08002816
2817 return 0;
2818}
2819
Eric Anholte47c68e2008-11-14 13:35:19 -08002820/**
2821 * Moves a single object to the CPU read, and possibly write domain.
2822 *
2823 * This function returns when the move is complete, including waiting on
2824 * flushes to occur.
2825 */
2826static int
2827i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
2828{
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002829 uint32_t old_write_domain, old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08002830 int ret;
2831
Daniel Vetterba3d8d72010-02-11 22:37:04 +01002832 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
Eric Anholte47c68e2008-11-14 13:35:19 -08002833 if (ret != 0)
2834 return ret;
2835
2836 i915_gem_object_flush_gtt_write_domain(obj);
2837
2838 /* If we have a partially-valid cache of the object in the CPU,
2839 * finish invalidating it and free the per-page flags.
2840 */
2841 i915_gem_object_set_to_full_cpu_read_domain(obj);
2842
Chris Wilson72133422010-09-13 23:56:38 +01002843 if (write) {
Chris Wilson2cf34d72010-09-14 13:03:28 +01002844 ret = i915_gem_object_wait_rendering(obj, true);
Chris Wilson72133422010-09-13 23:56:38 +01002845 if (ret)
2846 return ret;
2847 }
2848
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002849 old_write_domain = obj->write_domain;
2850 old_read_domains = obj->read_domains;
2851
Eric Anholte47c68e2008-11-14 13:35:19 -08002852 /* Flush the CPU cache if it's still invalid. */
2853 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
2854 i915_gem_clflush_object(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08002855
2856 obj->read_domains |= I915_GEM_DOMAIN_CPU;
2857 }
2858
2859 /* It should now be out of any other write domains, and we can update
2860 * the domain values for our changes.
2861 */
2862 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
2863
2864 /* If we're writing through the CPU, then the GPU read domains will
2865 * need to be invalidated at next use.
2866 */
2867 if (write) {
Chris Wilsonc78ec302010-09-20 12:50:23 +01002868 obj->read_domains = I915_GEM_DOMAIN_CPU;
Eric Anholte47c68e2008-11-14 13:35:19 -08002869 obj->write_domain = I915_GEM_DOMAIN_CPU;
2870 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002871
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002872 trace_i915_gem_object_change_domain(obj,
2873 old_read_domains,
2874 old_write_domain);
2875
Eric Anholt2ef7eea2008-11-10 10:53:25 -08002876 return 0;
2877}
2878
Eric Anholt673a3942008-07-30 12:06:12 -07002879/*
2880 * Set the next domain for the specified object. This
2881 * may not actually perform the necessary flushing/invaliding though,
2882 * as that may want to be batched with other set_domain operations
2883 *
2884 * This is (we hope) the only really tricky part of gem. The goal
2885 * is fairly simple -- track which caches hold bits of the object
2886 * and make sure they remain coherent. A few concrete examples may
2887 * help to explain how it works. For shorthand, we use the notation
2888 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
2889 * a pair of read and write domain masks.
2890 *
2891 * Case 1: the batch buffer
2892 *
2893 * 1. Allocated
2894 * 2. Written by CPU
2895 * 3. Mapped to GTT
2896 * 4. Read by GPU
2897 * 5. Unmapped from GTT
2898 * 6. Freed
2899 *
2900 * Let's take these a step at a time
2901 *
2902 * 1. Allocated
2903 * Pages allocated from the kernel may still have
2904 * cache contents, so we set them to (CPU, CPU) always.
2905 * 2. Written by CPU (using pwrite)
2906 * The pwrite function calls set_domain (CPU, CPU) and
2907 * this function does nothing (as nothing changes)
2908 * 3. Mapped by GTT
2909 * This function asserts that the object is not
2910 * currently in any GPU-based read or write domains
2911 * 4. Read by GPU
2912 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
2913 * As write_domain is zero, this function adds in the
2914 * current read domains (CPU+COMMAND, 0).
2915 * flush_domains is set to CPU.
2916 * invalidate_domains is set to COMMAND
2917 * clflush is run to get data out of the CPU caches
2918 * then i915_dev_set_domain calls i915_gem_flush to
2919 * emit an MI_FLUSH and drm_agp_chipset_flush
2920 * 5. Unmapped from GTT
2921 * i915_gem_object_unbind calls set_domain (CPU, CPU)
2922 * flush_domains and invalidate_domains end up both zero
2923 * so no flushing/invalidating happens
2924 * 6. Freed
2925 * yay, done
2926 *
2927 * Case 2: The shared render buffer
2928 *
2929 * 1. Allocated
2930 * 2. Mapped to GTT
2931 * 3. Read/written by GPU
2932 * 4. set_domain to (CPU,CPU)
2933 * 5. Read/written by CPU
2934 * 6. Read/written by GPU
2935 *
2936 * 1. Allocated
2937 * Same as last example, (CPU, CPU)
2938 * 2. Mapped to GTT
2939 * Nothing changes (assertions find that it is not in the GPU)
2940 * 3. Read/written by GPU
2941 * execbuffer calls set_domain (RENDER, RENDER)
2942 * flush_domains gets CPU
2943 * invalidate_domains gets GPU
2944 * clflush (obj)
2945 * MI_FLUSH and drm_agp_chipset_flush
2946 * 4. set_domain (CPU, CPU)
2947 * flush_domains gets GPU
2948 * invalidate_domains gets CPU
2949 * wait_rendering (obj) to make sure all drawing is complete.
2950 * This will include an MI_FLUSH to get the data from GPU
2951 * to memory
2952 * clflush (obj) to invalidate the CPU cache
2953 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
2954 * 5. Read/written by CPU
2955 * cache lines are loaded and dirtied
2956 * 6. Read written by GPU
2957 * Same as last GPU access
2958 *
2959 * Case 3: The constant buffer
2960 *
2961 * 1. Allocated
2962 * 2. Written by CPU
2963 * 3. Read by GPU
2964 * 4. Updated (written) by CPU again
2965 * 5. Read by GPU
2966 *
2967 * 1. Allocated
2968 * (CPU, CPU)
2969 * 2. Written by CPU
2970 * (CPU, CPU)
2971 * 3. Read by GPU
2972 * (CPU+RENDER, 0)
2973 * flush_domains = CPU
2974 * invalidate_domains = RENDER
2975 * clflush (obj)
2976 * MI_FLUSH
2977 * drm_agp_chipset_flush
2978 * 4. Updated (written) by CPU again
2979 * (CPU, CPU)
2980 * flush_domains = 0 (no previous write domain)
2981 * invalidate_domains = 0 (no new read domains)
2982 * 5. Read by GPU
2983 * (CPU+RENDER, 0)
2984 * flush_domains = CPU
2985 * invalidate_domains = RENDER
2986 * clflush (obj)
2987 * MI_FLUSH
2988 * drm_agp_chipset_flush
2989 */
Keith Packardc0d90822008-11-20 23:11:08 -08002990static void
Eric Anholt8b0e3782009-02-19 14:40:50 -08002991i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07002992{
2993 struct drm_device *dev = obj->dev;
Chris Wilson92204342010-09-18 11:02:01 +01002994 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01002995 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002996 uint32_t invalidate_domains = 0;
2997 uint32_t flush_domains = 0;
Chris Wilson1c5d22f2009-08-25 11:15:50 +01002998 uint32_t old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08002999
Eric Anholt8b0e3782009-02-19 14:40:50 -08003000 BUG_ON(obj->pending_read_domains & I915_GEM_DOMAIN_CPU);
3001 BUG_ON(obj->pending_write_domain == I915_GEM_DOMAIN_CPU);
Eric Anholt673a3942008-07-30 12:06:12 -07003002
Jesse Barnes652c3932009-08-17 13:31:43 -07003003 intel_mark_busy(dev, obj);
3004
Eric Anholt673a3942008-07-30 12:06:12 -07003005#if WATCH_BUF
3006 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
3007 __func__, obj,
Eric Anholt8b0e3782009-02-19 14:40:50 -08003008 obj->read_domains, obj->pending_read_domains,
3009 obj->write_domain, obj->pending_write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07003010#endif
3011 /*
3012 * If the object isn't moving to a new write domain,
3013 * let the object stay in multiple read domains
3014 */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003015 if (obj->pending_write_domain == 0)
3016 obj->pending_read_domains |= obj->read_domains;
Eric Anholt673a3942008-07-30 12:06:12 -07003017 else
3018 obj_priv->dirty = 1;
3019
3020 /*
3021 * Flush the current write domain if
3022 * the new read domains don't match. Invalidate
3023 * any read domains which differ from the old
3024 * write domain
3025 */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003026 if (obj->write_domain &&
3027 obj->write_domain != obj->pending_read_domains) {
Eric Anholt673a3942008-07-30 12:06:12 -07003028 flush_domains |= obj->write_domain;
Eric Anholt8b0e3782009-02-19 14:40:50 -08003029 invalidate_domains |=
3030 obj->pending_read_domains & ~obj->write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -07003031 }
3032 /*
3033 * Invalidate any read caches which may have
3034 * stale data. That is, any new read domains.
3035 */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003036 invalidate_domains |= obj->pending_read_domains & ~obj->read_domains;
Eric Anholt673a3942008-07-30 12:06:12 -07003037 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
3038#if WATCH_BUF
3039 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
3040 __func__, flush_domains, invalidate_domains);
3041#endif
Eric Anholt673a3942008-07-30 12:06:12 -07003042 i915_gem_clflush_object(obj);
3043 }
3044
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003045 old_read_domains = obj->read_domains;
3046
Eric Anholtefbeed92009-02-19 14:54:51 -08003047 /* The actual obj->write_domain will be updated with
3048 * pending_write_domain after we emit the accumulated flush for all
3049 * of our domain changes in execbuffers (which clears objects'
3050 * write_domains). So if we have a current write domain that we
3051 * aren't changing, set pending_write_domain to that.
3052 */
3053 if (flush_domains == 0 && obj->pending_write_domain == 0)
3054 obj->pending_write_domain = obj->write_domain;
Eric Anholt8b0e3782009-02-19 14:40:50 -08003055 obj->read_domains = obj->pending_read_domains;
Eric Anholt673a3942008-07-30 12:06:12 -07003056
3057 dev->invalidate_domains |= invalidate_domains;
3058 dev->flush_domains |= flush_domains;
Chris Wilson92204342010-09-18 11:02:01 +01003059 if (obj_priv->ring)
3060 dev_priv->mm.flush_rings |= obj_priv->ring->id;
Eric Anholt673a3942008-07-30 12:06:12 -07003061#if WATCH_BUF
3062 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
3063 __func__,
3064 obj->read_domains, obj->write_domain,
3065 dev->invalidate_domains, dev->flush_domains);
3066#endif
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003067
3068 trace_i915_gem_object_change_domain(obj,
3069 old_read_domains,
3070 obj->write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07003071}
3072
3073/**
Eric Anholte47c68e2008-11-14 13:35:19 -08003074 * Moves the object from a partially CPU read to a full one.
Eric Anholt673a3942008-07-30 12:06:12 -07003075 *
Eric Anholte47c68e2008-11-14 13:35:19 -08003076 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3077 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3078 */
3079static void
3080i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
3081{
Daniel Vetter23010e42010-03-08 13:35:02 +01003082 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08003083
3084 if (!obj_priv->page_cpu_valid)
3085 return;
3086
3087 /* If we're partially in the CPU read domain, finish moving it in.
3088 */
3089 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
3090 int i;
3091
3092 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
3093 if (obj_priv->page_cpu_valid[i])
3094 continue;
Eric Anholt856fa192009-03-19 14:10:50 -07003095 drm_clflush_pages(obj_priv->pages + i, 1);
Eric Anholte47c68e2008-11-14 13:35:19 -08003096 }
Eric Anholte47c68e2008-11-14 13:35:19 -08003097 }
3098
3099 /* Free the page_cpu_valid mappings which are now stale, whether
3100 * or not we've got I915_GEM_DOMAIN_CPU.
3101 */
Eric Anholt9a298b22009-03-24 12:23:04 -07003102 kfree(obj_priv->page_cpu_valid);
Eric Anholte47c68e2008-11-14 13:35:19 -08003103 obj_priv->page_cpu_valid = NULL;
3104}
3105
3106/**
3107 * Set the CPU read domain on a range of the object.
3108 *
3109 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3110 * not entirely valid. The page_cpu_valid member of the object flags which
3111 * pages have been flushed, and will be respected by
3112 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3113 * of the whole object.
3114 *
3115 * This function returns when the move is complete, including waiting on
3116 * flushes to occur.
Eric Anholt673a3942008-07-30 12:06:12 -07003117 */
3118static int
Eric Anholte47c68e2008-11-14 13:35:19 -08003119i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
3120 uint64_t offset, uint64_t size)
Eric Anholt673a3942008-07-30 12:06:12 -07003121{
Daniel Vetter23010e42010-03-08 13:35:02 +01003122 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003123 uint32_t old_read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003124 int i, ret;
Eric Anholt673a3942008-07-30 12:06:12 -07003125
Eric Anholte47c68e2008-11-14 13:35:19 -08003126 if (offset == 0 && size == obj->size)
3127 return i915_gem_object_set_to_cpu_domain(obj, 0);
3128
Daniel Vetterba3d8d72010-02-11 22:37:04 +01003129 ret = i915_gem_object_flush_gpu_write_domain(obj, false);
Eric Anholte47c68e2008-11-14 13:35:19 -08003130 if (ret != 0)
3131 return ret;
3132 i915_gem_object_flush_gtt_write_domain(obj);
3133
3134 /* If we're already fully in the CPU read domain, we're done. */
3135 if (obj_priv->page_cpu_valid == NULL &&
3136 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07003137 return 0;
3138
Eric Anholte47c68e2008-11-14 13:35:19 -08003139 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3140 * newly adding I915_GEM_DOMAIN_CPU
3141 */
Eric Anholt673a3942008-07-30 12:06:12 -07003142 if (obj_priv->page_cpu_valid == NULL) {
Eric Anholt9a298b22009-03-24 12:23:04 -07003143 obj_priv->page_cpu_valid = kzalloc(obj->size / PAGE_SIZE,
3144 GFP_KERNEL);
Eric Anholte47c68e2008-11-14 13:35:19 -08003145 if (obj_priv->page_cpu_valid == NULL)
3146 return -ENOMEM;
3147 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
3148 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
Eric Anholt673a3942008-07-30 12:06:12 -07003149
3150 /* Flush the cache on any pages that are still invalid from the CPU's
3151 * perspective.
3152 */
Eric Anholte47c68e2008-11-14 13:35:19 -08003153 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3154 i++) {
Eric Anholt673a3942008-07-30 12:06:12 -07003155 if (obj_priv->page_cpu_valid[i])
3156 continue;
3157
Eric Anholt856fa192009-03-19 14:10:50 -07003158 drm_clflush_pages(obj_priv->pages + i, 1);
Eric Anholt673a3942008-07-30 12:06:12 -07003159
3160 obj_priv->page_cpu_valid[i] = 1;
3161 }
3162
Eric Anholte47c68e2008-11-14 13:35:19 -08003163 /* It should now be out of any other write domains, and we can update
3164 * the domain values for our changes.
3165 */
3166 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3167
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003168 old_read_domains = obj->read_domains;
Eric Anholte47c68e2008-11-14 13:35:19 -08003169 obj->read_domains |= I915_GEM_DOMAIN_CPU;
3170
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003171 trace_i915_gem_object_change_domain(obj,
3172 old_read_domains,
3173 obj->write_domain);
3174
Eric Anholt673a3942008-07-30 12:06:12 -07003175 return 0;
3176}
3177
3178/**
Eric Anholt673a3942008-07-30 12:06:12 -07003179 * Pin an object to the GTT and evaluate the relocations landing in it.
3180 */
3181static int
3182i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
3183 struct drm_file *file_priv,
Jesse Barnes76446ca2009-12-17 22:05:42 -05003184 struct drm_i915_gem_exec_object2 *entry,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003185 struct drm_i915_gem_relocation_entry *relocs)
Eric Anholt673a3942008-07-30 12:06:12 -07003186{
3187 struct drm_device *dev = obj->dev;
Keith Packard0839ccb2008-10-30 19:38:48 -07003188 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01003189 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003190 int i, ret;
Keith Packard0839ccb2008-10-30 19:38:48 -07003191 void __iomem *reloc_page;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003192 bool need_fence;
3193
3194 need_fence = entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
3195 obj_priv->tiling_mode != I915_TILING_NONE;
3196
3197 /* Check fence reg constraints and rebind if necessary */
Chris Wilson808b24d2010-05-27 13:18:15 +01003198 if (need_fence &&
3199 !i915_gem_object_fence_offset_ok(obj,
3200 obj_priv->tiling_mode)) {
3201 ret = i915_gem_object_unbind(obj);
3202 if (ret)
3203 return ret;
3204 }
Eric Anholt673a3942008-07-30 12:06:12 -07003205
3206 /* Choose the GTT offset for our buffer and put it there. */
3207 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
3208 if (ret)
3209 return ret;
3210
Jesse Barnes76446ca2009-12-17 22:05:42 -05003211 /*
3212 * Pre-965 chips need a fence register set up in order to
3213 * properly handle blits to/from tiled surfaces.
3214 */
3215 if (need_fence) {
Chris Wilson53640e12010-09-20 11:40:50 +01003216 ret = i915_gem_object_get_fence_reg(obj, true);
Jesse Barnes76446ca2009-12-17 22:05:42 -05003217 if (ret != 0) {
Jesse Barnes76446ca2009-12-17 22:05:42 -05003218 i915_gem_object_unpin(obj);
3219 return ret;
3220 }
Chris Wilson53640e12010-09-20 11:40:50 +01003221
3222 dev_priv->fence_regs[obj_priv->fence_reg].gpu = true;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003223 }
3224
Eric Anholt673a3942008-07-30 12:06:12 -07003225 entry->offset = obj_priv->gtt_offset;
3226
Eric Anholt673a3942008-07-30 12:06:12 -07003227 /* Apply the relocations, using the GTT aperture to avoid cache
3228 * flushing requirements.
3229 */
3230 for (i = 0; i < entry->relocation_count; i++) {
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003231 struct drm_i915_gem_relocation_entry *reloc= &relocs[i];
Eric Anholt673a3942008-07-30 12:06:12 -07003232 struct drm_gem_object *target_obj;
3233 struct drm_i915_gem_object *target_obj_priv;
Eric Anholt3043c602008-10-02 12:24:47 -07003234 uint32_t reloc_val, reloc_offset;
3235 uint32_t __iomem *reloc_entry;
Eric Anholt673a3942008-07-30 12:06:12 -07003236
Eric Anholt673a3942008-07-30 12:06:12 -07003237 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003238 reloc->target_handle);
Eric Anholt673a3942008-07-30 12:06:12 -07003239 if (target_obj == NULL) {
3240 i915_gem_object_unpin(obj);
Chris Wilsonbf79cb92010-08-04 14:19:46 +01003241 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07003242 }
Daniel Vetter23010e42010-03-08 13:35:02 +01003243 target_obj_priv = to_intel_bo(target_obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003244
Chris Wilson8542a0b2009-09-09 21:15:15 +01003245#if WATCH_RELOC
3246 DRM_INFO("%s: obj %p offset %08x target %d "
3247 "read %08x write %08x gtt %08x "
3248 "presumed %08x delta %08x\n",
3249 __func__,
3250 obj,
3251 (int) reloc->offset,
3252 (int) reloc->target_handle,
3253 (int) reloc->read_domains,
3254 (int) reloc->write_domain,
3255 (int) target_obj_priv->gtt_offset,
3256 (int) reloc->presumed_offset,
3257 reloc->delta);
3258#endif
3259
Eric Anholt673a3942008-07-30 12:06:12 -07003260 /* The target buffer should have appeared before us in the
3261 * exec_object list, so it should have a GTT space bound by now.
3262 */
3263 if (target_obj_priv->gtt_space == NULL) {
3264 DRM_ERROR("No GTT space found for object %d\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003265 reloc->target_handle);
Eric Anholt673a3942008-07-30 12:06:12 -07003266 drm_gem_object_unreference(target_obj);
3267 i915_gem_object_unpin(obj);
3268 return -EINVAL;
3269 }
3270
Chris Wilson8542a0b2009-09-09 21:15:15 +01003271 /* Validate that the target is in a valid r/w GPU domain */
Daniel Vetter16edd552010-02-19 11:52:02 +01003272 if (reloc->write_domain & (reloc->write_domain - 1)) {
3273 DRM_ERROR("reloc with multiple write domains: "
3274 "obj %p target %d offset %d "
3275 "read %08x write %08x",
3276 obj, reloc->target_handle,
3277 (int) reloc->offset,
3278 reloc->read_domains,
3279 reloc->write_domain);
3280 return -EINVAL;
3281 }
Chris Wilson8542a0b2009-09-09 21:15:15 +01003282 if (reloc->write_domain & I915_GEM_DOMAIN_CPU ||
3283 reloc->read_domains & I915_GEM_DOMAIN_CPU) {
3284 DRM_ERROR("reloc with read/write CPU domains: "
3285 "obj %p target %d offset %d "
3286 "read %08x write %08x",
3287 obj, reloc->target_handle,
3288 (int) reloc->offset,
3289 reloc->read_domains,
3290 reloc->write_domain);
3291 drm_gem_object_unreference(target_obj);
3292 i915_gem_object_unpin(obj);
3293 return -EINVAL;
3294 }
3295 if (reloc->write_domain && target_obj->pending_write_domain &&
3296 reloc->write_domain != target_obj->pending_write_domain) {
3297 DRM_ERROR("Write domain conflict: "
3298 "obj %p target %d offset %d "
3299 "new %08x old %08x\n",
3300 obj, reloc->target_handle,
3301 (int) reloc->offset,
3302 reloc->write_domain,
3303 target_obj->pending_write_domain);
3304 drm_gem_object_unreference(target_obj);
3305 i915_gem_object_unpin(obj);
3306 return -EINVAL;
3307 }
3308
3309 target_obj->pending_read_domains |= reloc->read_domains;
3310 target_obj->pending_write_domain |= reloc->write_domain;
3311
3312 /* If the relocation already has the right value in it, no
3313 * more work needs to be done.
3314 */
3315 if (target_obj_priv->gtt_offset == reloc->presumed_offset) {
3316 drm_gem_object_unreference(target_obj);
3317 continue;
3318 }
3319
3320 /* Check that the relocation address is valid... */
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003321 if (reloc->offset > obj->size - 4) {
Eric Anholt673a3942008-07-30 12:06:12 -07003322 DRM_ERROR("Relocation beyond object bounds: "
3323 "obj %p target %d offset %d size %d.\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003324 obj, reloc->target_handle,
3325 (int) reloc->offset, (int) obj->size);
Eric Anholt673a3942008-07-30 12:06:12 -07003326 drm_gem_object_unreference(target_obj);
3327 i915_gem_object_unpin(obj);
3328 return -EINVAL;
3329 }
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003330 if (reloc->offset & 3) {
Eric Anholt673a3942008-07-30 12:06:12 -07003331 DRM_ERROR("Relocation not 4-byte aligned: "
3332 "obj %p target %d offset %d.\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003333 obj, reloc->target_handle,
3334 (int) reloc->offset);
Eric Anholt673a3942008-07-30 12:06:12 -07003335 drm_gem_object_unreference(target_obj);
3336 i915_gem_object_unpin(obj);
3337 return -EINVAL;
3338 }
3339
Chris Wilson8542a0b2009-09-09 21:15:15 +01003340 /* and points to somewhere within the target object. */
Chris Wilsoncd0b9fb2009-09-15 23:23:18 +01003341 if (reloc->delta >= target_obj->size) {
3342 DRM_ERROR("Relocation beyond target object bounds: "
3343 "obj %p target %d delta %d size %d.\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003344 obj, reloc->target_handle,
Chris Wilsoncd0b9fb2009-09-15 23:23:18 +01003345 (int) reloc->delta, (int) target_obj->size);
Chris Wilson491152b2009-02-11 14:26:32 +00003346 drm_gem_object_unreference(target_obj);
3347 i915_gem_object_unpin(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08003348 return -EINVAL;
3349 }
3350
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003351 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
3352 if (ret != 0) {
3353 drm_gem_object_unreference(target_obj);
3354 i915_gem_object_unpin(obj);
3355 return -EINVAL;
Eric Anholt673a3942008-07-30 12:06:12 -07003356 }
3357
3358 /* Map the page containing the relocation we're going to
3359 * perform.
3360 */
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003361 reloc_offset = obj_priv->gtt_offset + reloc->offset;
Keith Packard0839ccb2008-10-30 19:38:48 -07003362 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
3363 (reloc_offset &
Chris Wilsonfca3ec02010-08-04 14:34:24 +01003364 ~(PAGE_SIZE - 1)),
3365 KM_USER0);
Eric Anholt3043c602008-10-02 12:24:47 -07003366 reloc_entry = (uint32_t __iomem *)(reloc_page +
Keith Packard0839ccb2008-10-30 19:38:48 -07003367 (reloc_offset & (PAGE_SIZE - 1)));
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003368 reloc_val = target_obj_priv->gtt_offset + reloc->delta;
Eric Anholt673a3942008-07-30 12:06:12 -07003369
3370#if WATCH_BUF
3371 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003372 obj, (unsigned int) reloc->offset,
Eric Anholt673a3942008-07-30 12:06:12 -07003373 readl(reloc_entry), reloc_val);
3374#endif
3375 writel(reloc_val, reloc_entry);
Chris Wilsonfca3ec02010-08-04 14:34:24 +01003376 io_mapping_unmap_atomic(reloc_page, KM_USER0);
Eric Anholt673a3942008-07-30 12:06:12 -07003377
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003378 /* The updated presumed offset for this entry will be
3379 * copied back out to the user.
Eric Anholt673a3942008-07-30 12:06:12 -07003380 */
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003381 reloc->presumed_offset = target_obj_priv->gtt_offset;
Eric Anholt673a3942008-07-30 12:06:12 -07003382
3383 drm_gem_object_unreference(target_obj);
3384 }
3385
Eric Anholt673a3942008-07-30 12:06:12 -07003386#if WATCH_BUF
3387 if (0)
3388 i915_gem_dump_object(obj, 128, __func__, ~0);
3389#endif
3390 return 0;
3391}
3392
Eric Anholt673a3942008-07-30 12:06:12 -07003393/* Throttle our rendering by waiting until the ring has completed our requests
3394 * emitted over 20 msec ago.
3395 *
Eric Anholtb9624422009-06-03 07:27:35 +00003396 * Note that if we were to use the current jiffies each time around the loop,
3397 * we wouldn't escape the function with any frames outstanding if the time to
3398 * render a frame was over 20ms.
3399 *
Eric Anholt673a3942008-07-30 12:06:12 -07003400 * This should get us reasonable parallelism between CPU and GPU but also
3401 * relatively low latency when blocking on a particular request to finish.
3402 */
3403static int
3404i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
3405{
3406 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
3407 int ret = 0;
Eric Anholtb9624422009-06-03 07:27:35 +00003408 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
Eric Anholt673a3942008-07-30 12:06:12 -07003409
3410 mutex_lock(&dev->struct_mutex);
Eric Anholtb9624422009-06-03 07:27:35 +00003411 while (!list_empty(&i915_file_priv->mm.request_list)) {
3412 struct drm_i915_gem_request *request;
3413
3414 request = list_first_entry(&i915_file_priv->mm.request_list,
3415 struct drm_i915_gem_request,
3416 client_list);
3417
3418 if (time_after_eq(request->emitted_jiffies, recent_enough))
3419 break;
3420
Zou Nan hai852835f2010-05-21 09:08:56 +08003421 ret = i915_wait_request(dev, request->seqno, request->ring);
Eric Anholtb9624422009-06-03 07:27:35 +00003422 if (ret != 0)
3423 break;
3424 }
Eric Anholt673a3942008-07-30 12:06:12 -07003425 mutex_unlock(&dev->struct_mutex);
Eric Anholtb9624422009-06-03 07:27:35 +00003426
Eric Anholt673a3942008-07-30 12:06:12 -07003427 return ret;
3428}
3429
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003430static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003431i915_gem_get_relocs_from_user(struct drm_i915_gem_exec_object2 *exec_list,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003432 uint32_t buffer_count,
3433 struct drm_i915_gem_relocation_entry **relocs)
3434{
3435 uint32_t reloc_count = 0, reloc_index = 0, i;
3436 int ret;
3437
3438 *relocs = NULL;
3439 for (i = 0; i < buffer_count; i++) {
3440 if (reloc_count + exec_list[i].relocation_count < reloc_count)
3441 return -EINVAL;
3442 reloc_count += exec_list[i].relocation_count;
3443 }
3444
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003445 *relocs = drm_calloc_large(reloc_count, sizeof(**relocs));
Jesse Barnes76446ca2009-12-17 22:05:42 -05003446 if (*relocs == NULL) {
3447 DRM_ERROR("failed to alloc relocs, count %d\n", reloc_count);
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003448 return -ENOMEM;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003449 }
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003450
3451 for (i = 0; i < buffer_count; i++) {
3452 struct drm_i915_gem_relocation_entry __user *user_relocs;
3453
3454 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3455
3456 ret = copy_from_user(&(*relocs)[reloc_index],
3457 user_relocs,
3458 exec_list[i].relocation_count *
3459 sizeof(**relocs));
3460 if (ret != 0) {
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003461 drm_free_large(*relocs);
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003462 *relocs = NULL;
Florian Mickler2bc43b52009-04-06 22:55:41 +02003463 return -EFAULT;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003464 }
3465
3466 reloc_index += exec_list[i].relocation_count;
3467 }
3468
Florian Mickler2bc43b52009-04-06 22:55:41 +02003469 return 0;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003470}
3471
3472static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003473i915_gem_put_relocs_to_user(struct drm_i915_gem_exec_object2 *exec_list,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003474 uint32_t buffer_count,
3475 struct drm_i915_gem_relocation_entry *relocs)
3476{
3477 uint32_t reloc_count = 0, i;
Florian Mickler2bc43b52009-04-06 22:55:41 +02003478 int ret = 0;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003479
Chris Wilson93533c22010-01-31 10:40:48 +00003480 if (relocs == NULL)
3481 return 0;
3482
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003483 for (i = 0; i < buffer_count; i++) {
3484 struct drm_i915_gem_relocation_entry __user *user_relocs;
Florian Mickler2bc43b52009-04-06 22:55:41 +02003485 int unwritten;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003486
3487 user_relocs = (void __user *)(uintptr_t)exec_list[i].relocs_ptr;
3488
Florian Mickler2bc43b52009-04-06 22:55:41 +02003489 unwritten = copy_to_user(user_relocs,
3490 &relocs[reloc_count],
3491 exec_list[i].relocation_count *
3492 sizeof(*relocs));
3493
3494 if (unwritten) {
3495 ret = -EFAULT;
3496 goto err;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003497 }
3498
3499 reloc_count += exec_list[i].relocation_count;
3500 }
3501
Florian Mickler2bc43b52009-04-06 22:55:41 +02003502err:
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003503 drm_free_large(relocs);
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003504
3505 return ret;
3506}
3507
Chris Wilson83d60792009-06-06 09:45:57 +01003508static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003509i915_gem_check_execbuffer (struct drm_i915_gem_execbuffer2 *exec,
Chris Wilson83d60792009-06-06 09:45:57 +01003510 uint64_t exec_offset)
3511{
3512 uint32_t exec_start, exec_len;
3513
3514 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
3515 exec_len = (uint32_t) exec->batch_len;
3516
3517 if ((exec_start | exec_len) & 0x7)
3518 return -EINVAL;
3519
3520 if (!exec_start)
3521 return -EINVAL;
3522
3523 return 0;
3524}
3525
Chris Wilson265db952010-09-20 15:41:01 +01003526int
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003527i915_gem_wait_for_pending_flip(struct drm_device *dev,
3528 struct drm_gem_object **object_list,
3529 int count)
3530{
3531 drm_i915_private_t *dev_priv = dev->dev_private;
3532 struct drm_i915_gem_object *obj_priv;
3533 DEFINE_WAIT(wait);
3534 int i, ret = 0;
3535
3536 for (;;) {
3537 prepare_to_wait(&dev_priv->pending_flip_queue,
3538 &wait, TASK_INTERRUPTIBLE);
3539 for (i = 0; i < count; i++) {
Daniel Vetter23010e42010-03-08 13:35:02 +01003540 obj_priv = to_intel_bo(object_list[i]);
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003541 if (atomic_read(&obj_priv->pending_flip) > 0)
3542 break;
3543 }
3544 if (i == count)
3545 break;
3546
3547 if (!signal_pending(current)) {
3548 mutex_unlock(&dev->struct_mutex);
3549 schedule();
3550 mutex_lock(&dev->struct_mutex);
3551 continue;
3552 }
3553 ret = -ERESTARTSYS;
3554 break;
3555 }
3556 finish_wait(&dev_priv->pending_flip_queue, &wait);
3557
3558 return ret;
3559}
3560
Chris Wilson8dc5d142010-08-12 12:36:12 +01003561static int
Jesse Barnes76446ca2009-12-17 22:05:42 -05003562i915_gem_do_execbuffer(struct drm_device *dev, void *data,
3563 struct drm_file *file_priv,
3564 struct drm_i915_gem_execbuffer2 *args,
3565 struct drm_i915_gem_exec_object2 *exec_list)
Eric Anholt673a3942008-07-30 12:06:12 -07003566{
3567 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07003568 struct drm_gem_object **object_list = NULL;
3569 struct drm_gem_object *batch_obj;
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003570 struct drm_i915_gem_object *obj_priv;
Eric Anholt201361a2009-03-11 12:30:04 -07003571 struct drm_clip_rect *cliprects = NULL;
Chris Wilson93533c22010-01-31 10:40:48 +00003572 struct drm_i915_gem_relocation_entry *relocs = NULL;
Chris Wilson8dc5d142010-08-12 12:36:12 +01003573 struct drm_i915_gem_request *request = NULL;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003574 int ret = 0, ret2, i, pinned = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07003575 uint64_t exec_offset;
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01003576 uint32_t seqno, reloc_index;
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003577 int pin_tries, flips;
Eric Anholt673a3942008-07-30 12:06:12 -07003578
Zou Nan hai852835f2010-05-21 09:08:56 +08003579 struct intel_ring_buffer *ring = NULL;
3580
Eric Anholt673a3942008-07-30 12:06:12 -07003581#if WATCH_EXEC
3582 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3583 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3584#endif
Zou Nan haid1b851f2010-05-21 09:08:57 +08003585 if (args->flags & I915_EXEC_BSD) {
3586 if (!HAS_BSD(dev)) {
3587 DRM_ERROR("execbuf with wrong flag\n");
3588 return -EINVAL;
3589 }
3590 ring = &dev_priv->bsd_ring;
3591 } else {
3592 ring = &dev_priv->render_ring;
3593 }
3594
Eric Anholt4f481ed2008-09-10 14:22:49 -07003595 if (args->buffer_count < 1) {
3596 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3597 return -EINVAL;
3598 }
Eric Anholtc8e0f932009-11-22 03:49:37 +01003599 object_list = drm_malloc_ab(sizeof(*object_list), args->buffer_count);
Jesse Barnes76446ca2009-12-17 22:05:42 -05003600 if (object_list == NULL) {
3601 DRM_ERROR("Failed to allocate object list for %d buffers\n",
Eric Anholt673a3942008-07-30 12:06:12 -07003602 args->buffer_count);
3603 ret = -ENOMEM;
3604 goto pre_mutex_err;
3605 }
Eric Anholt673a3942008-07-30 12:06:12 -07003606
Eric Anholt201361a2009-03-11 12:30:04 -07003607 if (args->num_cliprects != 0) {
Eric Anholt9a298b22009-03-24 12:23:04 -07003608 cliprects = kcalloc(args->num_cliprects, sizeof(*cliprects),
3609 GFP_KERNEL);
Owain Ainswortha40e8d32010-02-09 14:25:55 +00003610 if (cliprects == NULL) {
3611 ret = -ENOMEM;
Eric Anholt201361a2009-03-11 12:30:04 -07003612 goto pre_mutex_err;
Owain Ainswortha40e8d32010-02-09 14:25:55 +00003613 }
Eric Anholt201361a2009-03-11 12:30:04 -07003614
3615 ret = copy_from_user(cliprects,
3616 (struct drm_clip_rect __user *)
3617 (uintptr_t) args->cliprects_ptr,
3618 sizeof(*cliprects) * args->num_cliprects);
3619 if (ret != 0) {
3620 DRM_ERROR("copy %d cliprects failed: %d\n",
3621 args->num_cliprects, ret);
Dan Carpenterc877cdc2010-06-23 19:03:01 +02003622 ret = -EFAULT;
Eric Anholt201361a2009-03-11 12:30:04 -07003623 goto pre_mutex_err;
3624 }
3625 }
3626
Chris Wilson8dc5d142010-08-12 12:36:12 +01003627 request = kzalloc(sizeof(*request), GFP_KERNEL);
3628 if (request == NULL) {
3629 ret = -ENOMEM;
3630 goto pre_mutex_err;
3631 }
3632
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003633 ret = i915_gem_get_relocs_from_user(exec_list, args->buffer_count,
3634 &relocs);
3635 if (ret != 0)
3636 goto pre_mutex_err;
3637
Eric Anholt673a3942008-07-30 12:06:12 -07003638 mutex_lock(&dev->struct_mutex);
3639
3640 i915_verify_inactive(dev, __FILE__, __LINE__);
3641
Ben Gamariba1234d2009-09-14 17:48:47 -04003642 if (atomic_read(&dev_priv->mm.wedged)) {
Eric Anholt673a3942008-07-30 12:06:12 -07003643 mutex_unlock(&dev->struct_mutex);
Chris Wilsona198bc82009-02-06 16:55:20 +00003644 ret = -EIO;
3645 goto pre_mutex_err;
Eric Anholt673a3942008-07-30 12:06:12 -07003646 }
3647
3648 if (dev_priv->mm.suspended) {
Eric Anholt673a3942008-07-30 12:06:12 -07003649 mutex_unlock(&dev->struct_mutex);
Chris Wilsona198bc82009-02-06 16:55:20 +00003650 ret = -EBUSY;
3651 goto pre_mutex_err;
Eric Anholt673a3942008-07-30 12:06:12 -07003652 }
3653
Keith Packardac94a962008-11-20 23:30:27 -08003654 /* Look up object handles */
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003655 flips = 0;
Eric Anholt673a3942008-07-30 12:06:12 -07003656 for (i = 0; i < args->buffer_count; i++) {
3657 object_list[i] = drm_gem_object_lookup(dev, file_priv,
3658 exec_list[i].handle);
3659 if (object_list[i] == NULL) {
3660 DRM_ERROR("Invalid object handle %d at index %d\n",
3661 exec_list[i].handle, i);
Chris Wilson0ce907f2010-01-23 20:26:35 +00003662 /* prevent error path from reading uninitialized data */
3663 args->buffer_count = i + 1;
Chris Wilsonbf79cb92010-08-04 14:19:46 +01003664 ret = -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07003665 goto err;
3666 }
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003667
Daniel Vetter23010e42010-03-08 13:35:02 +01003668 obj_priv = to_intel_bo(object_list[i]);
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003669 if (obj_priv->in_execbuffer) {
3670 DRM_ERROR("Object %p appears more than once in object list\n",
3671 object_list[i]);
Chris Wilson0ce907f2010-01-23 20:26:35 +00003672 /* prevent error path from reading uninitialized data */
3673 args->buffer_count = i + 1;
Chris Wilsonbf79cb92010-08-04 14:19:46 +01003674 ret = -EINVAL;
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003675 goto err;
3676 }
3677 obj_priv->in_execbuffer = true;
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05003678 flips += atomic_read(&obj_priv->pending_flip);
3679 }
3680
3681 if (flips > 0) {
3682 ret = i915_gem_wait_for_pending_flip(dev, object_list,
3683 args->buffer_count);
3684 if (ret)
3685 goto err;
Keith Packardac94a962008-11-20 23:30:27 -08003686 }
Eric Anholt673a3942008-07-30 12:06:12 -07003687
Keith Packardac94a962008-11-20 23:30:27 -08003688 /* Pin and relocate */
3689 for (pin_tries = 0; ; pin_tries++) {
3690 ret = 0;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003691 reloc_index = 0;
3692
Keith Packardac94a962008-11-20 23:30:27 -08003693 for (i = 0; i < args->buffer_count; i++) {
3694 object_list[i]->pending_read_domains = 0;
3695 object_list[i]->pending_write_domain = 0;
3696 ret = i915_gem_object_pin_and_relocate(object_list[i],
3697 file_priv,
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003698 &exec_list[i],
3699 &relocs[reloc_index]);
Keith Packardac94a962008-11-20 23:30:27 -08003700 if (ret)
3701 break;
3702 pinned = i + 1;
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003703 reloc_index += exec_list[i].relocation_count;
Keith Packardac94a962008-11-20 23:30:27 -08003704 }
3705 /* success */
3706 if (ret == 0)
3707 break;
3708
3709 /* error other than GTT full, or we've already tried again */
Chris Wilson2939e1f2009-06-06 09:46:03 +01003710 if (ret != -ENOSPC || pin_tries >= 1) {
Chris Wilson07f73f62009-09-14 16:50:30 +01003711 if (ret != -ERESTARTSYS) {
3712 unsigned long long total_size = 0;
Chris Wilson3d1cc472010-05-27 13:18:19 +01003713 int num_fences = 0;
3714 for (i = 0; i < args->buffer_count; i++) {
Chris Wilson43b27f42010-07-02 08:57:15 +01003715 obj_priv = to_intel_bo(object_list[i]);
Chris Wilson3d1cc472010-05-27 13:18:19 +01003716
Chris Wilson07f73f62009-09-14 16:50:30 +01003717 total_size += object_list[i]->size;
Chris Wilson3d1cc472010-05-27 13:18:19 +01003718 num_fences +=
3719 exec_list[i].flags & EXEC_OBJECT_NEEDS_FENCE &&
3720 obj_priv->tiling_mode != I915_TILING_NONE;
3721 }
3722 DRM_ERROR("Failed to pin buffer %d of %d, total %llu bytes, %d fences: %d\n",
Chris Wilson07f73f62009-09-14 16:50:30 +01003723 pinned+1, args->buffer_count,
Chris Wilson3d1cc472010-05-27 13:18:19 +01003724 total_size, num_fences,
3725 ret);
Chris Wilson07f73f62009-09-14 16:50:30 +01003726 DRM_ERROR("%d objects [%d pinned], "
3727 "%d object bytes [%d pinned], "
3728 "%d/%d gtt bytes\n",
3729 atomic_read(&dev->object_count),
3730 atomic_read(&dev->pin_count),
3731 atomic_read(&dev->object_memory),
3732 atomic_read(&dev->pin_memory),
3733 atomic_read(&dev->gtt_memory),
3734 dev->gtt_total);
3735 }
Eric Anholt673a3942008-07-30 12:06:12 -07003736 goto err;
3737 }
Keith Packardac94a962008-11-20 23:30:27 -08003738
3739 /* unpin all of our buffers */
3740 for (i = 0; i < pinned; i++)
3741 i915_gem_object_unpin(object_list[i]);
Eric Anholtb1177632008-12-10 10:09:41 -08003742 pinned = 0;
Keith Packardac94a962008-11-20 23:30:27 -08003743
3744 /* evict everyone we can from the aperture */
3745 ret = i915_gem_evict_everything(dev);
Chris Wilson07f73f62009-09-14 16:50:30 +01003746 if (ret && ret != -ENOSPC)
Keith Packardac94a962008-11-20 23:30:27 -08003747 goto err;
Eric Anholt673a3942008-07-30 12:06:12 -07003748 }
3749
3750 /* Set the pending read domains for the batch buffer to COMMAND */
3751 batch_obj = object_list[args->buffer_count-1];
Chris Wilson5f26a2c2009-06-06 09:45:58 +01003752 if (batch_obj->pending_write_domain) {
3753 DRM_ERROR("Attempting to use self-modifying batch buffer\n");
3754 ret = -EINVAL;
3755 goto err;
3756 }
3757 batch_obj->pending_read_domains |= I915_GEM_DOMAIN_COMMAND;
Eric Anholt673a3942008-07-30 12:06:12 -07003758
Chris Wilson83d60792009-06-06 09:45:57 +01003759 /* Sanity check the batch buffer, prior to moving objects */
3760 exec_offset = exec_list[args->buffer_count - 1].offset;
3761 ret = i915_gem_check_execbuffer (args, exec_offset);
3762 if (ret != 0) {
3763 DRM_ERROR("execbuf with invalid offset/length\n");
3764 goto err;
3765 }
3766
Eric Anholt673a3942008-07-30 12:06:12 -07003767 i915_verify_inactive(dev, __FILE__, __LINE__);
3768
Keith Packard646f0f62008-11-20 23:23:03 -08003769 /* Zero the global flush/invalidate flags. These
3770 * will be modified as new domains are computed
3771 * for each object
3772 */
3773 dev->invalidate_domains = 0;
3774 dev->flush_domains = 0;
Chris Wilson92204342010-09-18 11:02:01 +01003775 dev_priv->mm.flush_rings = 0;
Keith Packard646f0f62008-11-20 23:23:03 -08003776
Eric Anholt673a3942008-07-30 12:06:12 -07003777 for (i = 0; i < args->buffer_count; i++) {
3778 struct drm_gem_object *obj = object_list[i];
Eric Anholt673a3942008-07-30 12:06:12 -07003779
Keith Packard646f0f62008-11-20 23:23:03 -08003780 /* Compute new gpu domains and update invalidate/flush */
Eric Anholt8b0e3782009-02-19 14:40:50 -08003781 i915_gem_object_set_to_gpu_domain(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07003782 }
3783
3784 i915_verify_inactive(dev, __FILE__, __LINE__);
3785
Keith Packard646f0f62008-11-20 23:23:03 -08003786 if (dev->invalidate_domains | dev->flush_domains) {
3787#if WATCH_EXEC
3788 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
3789 __func__,
3790 dev->invalidate_domains,
3791 dev->flush_domains);
3792#endif
Chris Wilsonc78ec302010-09-20 12:50:23 +01003793 i915_gem_flush(dev, file_priv,
Keith Packard646f0f62008-11-20 23:23:03 -08003794 dev->invalidate_domains,
Chris Wilson92204342010-09-18 11:02:01 +01003795 dev->flush_domains,
3796 dev_priv->mm.flush_rings);
Daniel Vettera6910432010-02-02 17:08:37 +01003797 }
3798
Eric Anholtefbeed92009-02-19 14:54:51 -08003799 for (i = 0; i < args->buffer_count; i++) {
3800 struct drm_gem_object *obj = object_list[i];
Daniel Vetter23010e42010-03-08 13:35:02 +01003801 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003802 uint32_t old_write_domain = obj->write_domain;
Eric Anholtefbeed92009-02-19 14:54:51 -08003803
3804 obj->write_domain = obj->pending_write_domain;
Daniel Vetter99fcb762010-02-07 16:20:18 +01003805 if (obj->write_domain)
3806 list_move_tail(&obj_priv->gpu_write_list,
3807 &dev_priv->mm.gpu_write_list);
3808 else
3809 list_del_init(&obj_priv->gpu_write_list);
3810
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003811 trace_i915_gem_object_change_domain(obj,
3812 obj->read_domains,
3813 old_write_domain);
Eric Anholtefbeed92009-02-19 14:54:51 -08003814 }
3815
Eric Anholt673a3942008-07-30 12:06:12 -07003816 i915_verify_inactive(dev, __FILE__, __LINE__);
3817
3818#if WATCH_COHERENCY
3819 for (i = 0; i < args->buffer_count; i++) {
3820 i915_gem_object_check_coherency(object_list[i],
3821 exec_list[i].handle);
3822 }
3823#endif
3824
Eric Anholt673a3942008-07-30 12:06:12 -07003825#if WATCH_EXEC
Ben Gamari6911a9b2009-04-02 11:24:54 -07003826 i915_gem_dump_object(batch_obj,
Eric Anholt673a3942008-07-30 12:06:12 -07003827 args->batch_len,
3828 __func__,
3829 ~0);
3830#endif
3831
Eric Anholt673a3942008-07-30 12:06:12 -07003832 /* Exec the batchbuffer */
Zou Nan hai852835f2010-05-21 09:08:56 +08003833 ret = ring->dispatch_gem_execbuffer(dev, ring, args,
3834 cliprects, exec_offset);
Eric Anholt673a3942008-07-30 12:06:12 -07003835 if (ret) {
3836 DRM_ERROR("dispatch failed %d\n", ret);
3837 goto err;
3838 }
3839
3840 /*
3841 * Ensure that the commands in the batch buffer are
3842 * finished before the interrupt fires
3843 */
Daniel Vetter8a1a49f2010-02-11 22:29:04 +01003844 i915_retire_commands(dev, ring);
Eric Anholt673a3942008-07-30 12:06:12 -07003845
3846 i915_verify_inactive(dev, __FILE__, __LINE__);
3847
Daniel Vetter617dbe22010-02-11 22:16:02 +01003848 for (i = 0; i < args->buffer_count; i++) {
3849 struct drm_gem_object *obj = object_list[i];
3850 obj_priv = to_intel_bo(obj);
3851
3852 i915_gem_object_move_to_active(obj, ring);
3853#if WATCH_LRU
3854 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
3855#endif
3856 }
3857
Eric Anholt673a3942008-07-30 12:06:12 -07003858 /*
3859 * Get a seqno representing the execution of the current buffer,
3860 * which we can wait on. We would like to mitigate these interrupts,
3861 * likely by only creating seqnos occasionally (so that we have
3862 * *some* interrupts representing completion of buffers that we can
3863 * wait on when trying to clear up gtt space).
3864 */
Chris Wilson8dc5d142010-08-12 12:36:12 +01003865 seqno = i915_add_request(dev, file_priv, request, ring);
3866 request = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07003867
Eric Anholt673a3942008-07-30 12:06:12 -07003868#if WATCH_LRU
3869 i915_dump_lru(dev, __func__);
3870#endif
3871
3872 i915_verify_inactive(dev, __FILE__, __LINE__);
3873
Eric Anholt673a3942008-07-30 12:06:12 -07003874err:
Julia Lawallaad87df2008-12-21 16:28:47 +01003875 for (i = 0; i < pinned; i++)
3876 i915_gem_object_unpin(object_list[i]);
Eric Anholt673a3942008-07-30 12:06:12 -07003877
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003878 for (i = 0; i < args->buffer_count; i++) {
3879 if (object_list[i]) {
Daniel Vetter23010e42010-03-08 13:35:02 +01003880 obj_priv = to_intel_bo(object_list[i]);
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003881 obj_priv->in_execbuffer = false;
3882 }
Julia Lawallaad87df2008-12-21 16:28:47 +01003883 drm_gem_object_unreference(object_list[i]);
Kristian Høgsbergb70d11d2009-03-03 14:45:57 -05003884 }
Julia Lawallaad87df2008-12-21 16:28:47 +01003885
Eric Anholt673a3942008-07-30 12:06:12 -07003886 mutex_unlock(&dev->struct_mutex);
3887
Chris Wilson93533c22010-01-31 10:40:48 +00003888pre_mutex_err:
Eric Anholt40a5f0d2009-03-12 11:23:52 -07003889 /* Copy the updated relocations out regardless of current error
3890 * state. Failure to update the relocs would mean that the next
3891 * time userland calls execbuf, it would do so with presumed offset
3892 * state that didn't match the actual object state.
3893 */
3894 ret2 = i915_gem_put_relocs_to_user(exec_list, args->buffer_count,
3895 relocs);
3896 if (ret2 != 0) {
3897 DRM_ERROR("Failed to copy relocations back out: %d\n", ret2);
3898
3899 if (ret == 0)
3900 ret = ret2;
3901 }
3902
Jesse Barnes8e7d2b22009-05-08 16:13:25 -07003903 drm_free_large(object_list);
Eric Anholt9a298b22009-03-24 12:23:04 -07003904 kfree(cliprects);
Chris Wilson8dc5d142010-08-12 12:36:12 +01003905 kfree(request);
Eric Anholt673a3942008-07-30 12:06:12 -07003906
3907 return ret;
3908}
3909
Jesse Barnes76446ca2009-12-17 22:05:42 -05003910/*
3911 * Legacy execbuffer just creates an exec2 list from the original exec object
3912 * list array and passes it to the real function.
3913 */
3914int
3915i915_gem_execbuffer(struct drm_device *dev, void *data,
3916 struct drm_file *file_priv)
3917{
3918 struct drm_i915_gem_execbuffer *args = data;
3919 struct drm_i915_gem_execbuffer2 exec2;
3920 struct drm_i915_gem_exec_object *exec_list = NULL;
3921 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
3922 int ret, i;
3923
3924#if WATCH_EXEC
3925 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
3926 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
3927#endif
3928
3929 if (args->buffer_count < 1) {
3930 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
3931 return -EINVAL;
3932 }
3933
3934 /* Copy in the exec list from userland */
3935 exec_list = drm_malloc_ab(sizeof(*exec_list), args->buffer_count);
3936 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
3937 if (exec_list == NULL || exec2_list == NULL) {
3938 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
3939 args->buffer_count);
3940 drm_free_large(exec_list);
3941 drm_free_large(exec2_list);
3942 return -ENOMEM;
3943 }
3944 ret = copy_from_user(exec_list,
3945 (struct drm_i915_relocation_entry __user *)
3946 (uintptr_t) args->buffers_ptr,
3947 sizeof(*exec_list) * args->buffer_count);
3948 if (ret != 0) {
3949 DRM_ERROR("copy %d exec entries failed %d\n",
3950 args->buffer_count, ret);
3951 drm_free_large(exec_list);
3952 drm_free_large(exec2_list);
3953 return -EFAULT;
3954 }
3955
3956 for (i = 0; i < args->buffer_count; i++) {
3957 exec2_list[i].handle = exec_list[i].handle;
3958 exec2_list[i].relocation_count = exec_list[i].relocation_count;
3959 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr;
3960 exec2_list[i].alignment = exec_list[i].alignment;
3961 exec2_list[i].offset = exec_list[i].offset;
Chris Wilsona6c45cf2010-09-17 00:32:17 +01003962 if (INTEL_INFO(dev)->gen < 4)
Jesse Barnes76446ca2009-12-17 22:05:42 -05003963 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE;
3964 else
3965 exec2_list[i].flags = 0;
3966 }
3967
3968 exec2.buffers_ptr = args->buffers_ptr;
3969 exec2.buffer_count = args->buffer_count;
3970 exec2.batch_start_offset = args->batch_start_offset;
3971 exec2.batch_len = args->batch_len;
3972 exec2.DR1 = args->DR1;
3973 exec2.DR4 = args->DR4;
3974 exec2.num_cliprects = args->num_cliprects;
3975 exec2.cliprects_ptr = args->cliprects_ptr;
Zou Nan hai852835f2010-05-21 09:08:56 +08003976 exec2.flags = I915_EXEC_RENDER;
Jesse Barnes76446ca2009-12-17 22:05:42 -05003977
3978 ret = i915_gem_do_execbuffer(dev, data, file_priv, &exec2, exec2_list);
3979 if (!ret) {
3980 /* Copy the new buffer offsets back to the user's exec list. */
3981 for (i = 0; i < args->buffer_count; i++)
3982 exec_list[i].offset = exec2_list[i].offset;
3983 /* ... and back out to userspace */
3984 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
3985 (uintptr_t) args->buffers_ptr,
3986 exec_list,
3987 sizeof(*exec_list) * args->buffer_count);
3988 if (ret) {
3989 ret = -EFAULT;
3990 DRM_ERROR("failed to copy %d exec entries "
3991 "back to user (%d)\n",
3992 args->buffer_count, ret);
3993 }
Jesse Barnes76446ca2009-12-17 22:05:42 -05003994 }
3995
3996 drm_free_large(exec_list);
3997 drm_free_large(exec2_list);
3998 return ret;
3999}
4000
4001int
4002i915_gem_execbuffer2(struct drm_device *dev, void *data,
4003 struct drm_file *file_priv)
4004{
4005 struct drm_i915_gem_execbuffer2 *args = data;
4006 struct drm_i915_gem_exec_object2 *exec2_list = NULL;
4007 int ret;
4008
4009#if WATCH_EXEC
4010 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
4011 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
4012#endif
4013
4014 if (args->buffer_count < 1) {
4015 DRM_ERROR("execbuf2 with %d buffers\n", args->buffer_count);
4016 return -EINVAL;
4017 }
4018
4019 exec2_list = drm_malloc_ab(sizeof(*exec2_list), args->buffer_count);
4020 if (exec2_list == NULL) {
4021 DRM_ERROR("Failed to allocate exec list for %d buffers\n",
4022 args->buffer_count);
4023 return -ENOMEM;
4024 }
4025 ret = copy_from_user(exec2_list,
4026 (struct drm_i915_relocation_entry __user *)
4027 (uintptr_t) args->buffers_ptr,
4028 sizeof(*exec2_list) * args->buffer_count);
4029 if (ret != 0) {
4030 DRM_ERROR("copy %d exec entries failed %d\n",
4031 args->buffer_count, ret);
4032 drm_free_large(exec2_list);
4033 return -EFAULT;
4034 }
4035
4036 ret = i915_gem_do_execbuffer(dev, data, file_priv, args, exec2_list);
4037 if (!ret) {
4038 /* Copy the new buffer offsets back to the user's exec list. */
4039 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
4040 (uintptr_t) args->buffers_ptr,
4041 exec2_list,
4042 sizeof(*exec2_list) * args->buffer_count);
4043 if (ret) {
4044 ret = -EFAULT;
4045 DRM_ERROR("failed to copy %d exec entries "
4046 "back to user (%d)\n",
4047 args->buffer_count, ret);
4048 }
4049 }
4050
4051 drm_free_large(exec2_list);
4052 return ret;
4053}
4054
Eric Anholt673a3942008-07-30 12:06:12 -07004055int
4056i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
4057{
4058 struct drm_device *dev = obj->dev;
Chris Wilsonf13d3f72010-09-20 17:36:15 +01004059 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01004060 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004061 int ret;
4062
Daniel Vetter778c3542010-05-13 11:49:44 +02004063 BUG_ON(obj_priv->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
4064
Eric Anholt673a3942008-07-30 12:06:12 -07004065 i915_verify_inactive(dev, __FILE__, __LINE__);
Chris Wilsonac0c6b52010-05-27 13:18:18 +01004066
4067 if (obj_priv->gtt_space != NULL) {
4068 if (alignment == 0)
4069 alignment = i915_gem_get_gtt_alignment(obj);
4070 if (obj_priv->gtt_offset & (alignment - 1)) {
Chris Wilsonae7d49d2010-08-04 12:37:41 +01004071 WARN(obj_priv->pin_count,
4072 "bo is already pinned with incorrect alignment:"
4073 " offset=%x, req.alignment=%x\n",
4074 obj_priv->gtt_offset, alignment);
Chris Wilsonac0c6b52010-05-27 13:18:18 +01004075 ret = i915_gem_object_unbind(obj);
4076 if (ret)
4077 return ret;
4078 }
4079 }
4080
Eric Anholt673a3942008-07-30 12:06:12 -07004081 if (obj_priv->gtt_space == NULL) {
4082 ret = i915_gem_object_bind_to_gtt(obj, alignment);
Chris Wilson97311292009-09-21 00:22:34 +01004083 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -07004084 return ret;
Chris Wilson22c344e2009-02-11 14:26:45 +00004085 }
Jesse Barnes76446ca2009-12-17 22:05:42 -05004086
Eric Anholt673a3942008-07-30 12:06:12 -07004087 obj_priv->pin_count++;
4088
4089 /* If the object is not active and not pending a flush,
4090 * remove it from the inactive list
4091 */
4092 if (obj_priv->pin_count == 1) {
4093 atomic_inc(&dev->pin_count);
4094 atomic_add(obj->size, &dev->pin_memory);
Chris Wilsonf13d3f72010-09-20 17:36:15 +01004095 if (!obj_priv->active)
4096 list_move_tail(&obj_priv->list,
4097 &dev_priv->mm.pinned_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004098 }
4099 i915_verify_inactive(dev, __FILE__, __LINE__);
4100
4101 return 0;
4102}
4103
4104void
4105i915_gem_object_unpin(struct drm_gem_object *obj)
4106{
4107 struct drm_device *dev = obj->dev;
4108 drm_i915_private_t *dev_priv = dev->dev_private;
Daniel Vetter23010e42010-03-08 13:35:02 +01004109 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004110
4111 i915_verify_inactive(dev, __FILE__, __LINE__);
4112 obj_priv->pin_count--;
4113 BUG_ON(obj_priv->pin_count < 0);
4114 BUG_ON(obj_priv->gtt_space == NULL);
4115
4116 /* If the object is no longer pinned, and is
4117 * neither active nor being flushed, then stick it on
4118 * the inactive list
4119 */
4120 if (obj_priv->pin_count == 0) {
Chris Wilsonf13d3f72010-09-20 17:36:15 +01004121 if (!obj_priv->active)
Eric Anholt673a3942008-07-30 12:06:12 -07004122 list_move_tail(&obj_priv->list,
4123 &dev_priv->mm.inactive_list);
4124 atomic_dec(&dev->pin_count);
4125 atomic_sub(obj->size, &dev->pin_memory);
4126 }
4127 i915_verify_inactive(dev, __FILE__, __LINE__);
4128}
4129
4130int
4131i915_gem_pin_ioctl(struct drm_device *dev, void *data,
4132 struct drm_file *file_priv)
4133{
4134 struct drm_i915_gem_pin *args = data;
4135 struct drm_gem_object *obj;
4136 struct drm_i915_gem_object *obj_priv;
4137 int ret;
4138
4139 mutex_lock(&dev->struct_mutex);
4140
4141 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4142 if (obj == NULL) {
4143 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
4144 args->handle);
4145 mutex_unlock(&dev->struct_mutex);
Chris Wilsonbf79cb92010-08-04 14:19:46 +01004146 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07004147 }
Daniel Vetter23010e42010-03-08 13:35:02 +01004148 obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004149
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004150 if (obj_priv->madv != I915_MADV_WILLNEED) {
4151 DRM_ERROR("Attempting to pin a purgeable buffer\n");
Chris Wilson3ef94da2009-09-14 16:50:29 +01004152 drm_gem_object_unreference(obj);
4153 mutex_unlock(&dev->struct_mutex);
4154 return -EINVAL;
4155 }
4156
Jesse Barnes79e53942008-11-07 14:24:08 -08004157 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != file_priv) {
4158 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
4159 args->handle);
Chris Wilson96dec612009-02-08 19:08:04 +00004160 drm_gem_object_unreference(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004161 mutex_unlock(&dev->struct_mutex);
Jesse Barnes79e53942008-11-07 14:24:08 -08004162 return -EINVAL;
4163 }
4164
4165 obj_priv->user_pin_count++;
4166 obj_priv->pin_filp = file_priv;
4167 if (obj_priv->user_pin_count == 1) {
4168 ret = i915_gem_object_pin(obj, args->alignment);
4169 if (ret != 0) {
4170 drm_gem_object_unreference(obj);
4171 mutex_unlock(&dev->struct_mutex);
4172 return ret;
4173 }
Eric Anholt673a3942008-07-30 12:06:12 -07004174 }
4175
4176 /* XXX - flush the CPU caches for pinned objects
4177 * as the X server doesn't manage domains yet
4178 */
Eric Anholte47c68e2008-11-14 13:35:19 -08004179 i915_gem_object_flush_cpu_write_domain(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004180 args->offset = obj_priv->gtt_offset;
4181 drm_gem_object_unreference(obj);
4182 mutex_unlock(&dev->struct_mutex);
4183
4184 return 0;
4185}
4186
4187int
4188i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
4189 struct drm_file *file_priv)
4190{
4191 struct drm_i915_gem_pin *args = data;
4192 struct drm_gem_object *obj;
Jesse Barnes79e53942008-11-07 14:24:08 -08004193 struct drm_i915_gem_object *obj_priv;
Eric Anholt673a3942008-07-30 12:06:12 -07004194
4195 mutex_lock(&dev->struct_mutex);
4196
4197 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4198 if (obj == NULL) {
4199 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
4200 args->handle);
4201 mutex_unlock(&dev->struct_mutex);
Chris Wilsonbf79cb92010-08-04 14:19:46 +01004202 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07004203 }
4204
Daniel Vetter23010e42010-03-08 13:35:02 +01004205 obj_priv = to_intel_bo(obj);
Jesse Barnes79e53942008-11-07 14:24:08 -08004206 if (obj_priv->pin_filp != file_priv) {
4207 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
4208 args->handle);
4209 drm_gem_object_unreference(obj);
4210 mutex_unlock(&dev->struct_mutex);
4211 return -EINVAL;
4212 }
4213 obj_priv->user_pin_count--;
4214 if (obj_priv->user_pin_count == 0) {
4215 obj_priv->pin_filp = NULL;
4216 i915_gem_object_unpin(obj);
4217 }
Eric Anholt673a3942008-07-30 12:06:12 -07004218
4219 drm_gem_object_unreference(obj);
4220 mutex_unlock(&dev->struct_mutex);
4221 return 0;
4222}
4223
4224int
4225i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4226 struct drm_file *file_priv)
4227{
4228 struct drm_i915_gem_busy *args = data;
4229 struct drm_gem_object *obj;
4230 struct drm_i915_gem_object *obj_priv;
4231
Eric Anholt673a3942008-07-30 12:06:12 -07004232 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4233 if (obj == NULL) {
4234 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
4235 args->handle);
Chris Wilsonbf79cb92010-08-04 14:19:46 +01004236 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07004237 }
4238
Chris Wilsonb1ce7862009-06-06 09:46:00 +01004239 mutex_lock(&dev->struct_mutex);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004240
Chris Wilson0be555b2010-08-04 15:36:30 +01004241 /* Count all active objects as busy, even if they are currently not used
4242 * by the gpu. Users of this interface expect objects to eventually
4243 * become non-busy without any further actions, therefore emit any
4244 * necessary flushes here.
Eric Anholtc4de0a52008-12-14 19:05:04 -08004245 */
Chris Wilson0be555b2010-08-04 15:36:30 +01004246 obj_priv = to_intel_bo(obj);
4247 args->busy = obj_priv->active;
4248 if (args->busy) {
4249 /* Unconditionally flush objects, even when the gpu still uses this
4250 * object. Userspace calling this function indicates that it wants to
4251 * use this buffer rather sooner than later, so issuing the required
4252 * flush earlier is beneficial.
4253 */
Chris Wilsonc78ec302010-09-20 12:50:23 +01004254 if (obj->write_domain & I915_GEM_GPU_DOMAINS)
4255 i915_gem_flush_ring(dev, file_priv,
Chris Wilson92204342010-09-18 11:02:01 +01004256 obj_priv->ring,
4257 0, obj->write_domain);
Chris Wilson0be555b2010-08-04 15:36:30 +01004258
4259 /* Update the active list for the hardware's current position.
4260 * Otherwise this only updates on a delayed timer or when irqs
4261 * are actually unmasked, and our working set ends up being
4262 * larger than required.
4263 */
4264 i915_gem_retire_requests_ring(dev, obj_priv->ring);
4265
4266 args->busy = obj_priv->active;
4267 }
Eric Anholt673a3942008-07-30 12:06:12 -07004268
4269 drm_gem_object_unreference(obj);
4270 mutex_unlock(&dev->struct_mutex);
4271 return 0;
4272}
4273
4274int
4275i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4276 struct drm_file *file_priv)
4277{
4278 return i915_gem_ring_throttle(dev, file_priv);
4279}
4280
Chris Wilson3ef94da2009-09-14 16:50:29 +01004281int
4282i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4283 struct drm_file *file_priv)
4284{
4285 struct drm_i915_gem_madvise *args = data;
4286 struct drm_gem_object *obj;
4287 struct drm_i915_gem_object *obj_priv;
4288
4289 switch (args->madv) {
4290 case I915_MADV_DONTNEED:
4291 case I915_MADV_WILLNEED:
4292 break;
4293 default:
4294 return -EINVAL;
4295 }
4296
4297 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
4298 if (obj == NULL) {
4299 DRM_ERROR("Bad handle in i915_gem_madvise_ioctl(): %d\n",
4300 args->handle);
Chris Wilsonbf79cb92010-08-04 14:19:46 +01004301 return -ENOENT;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004302 }
4303
4304 mutex_lock(&dev->struct_mutex);
Daniel Vetter23010e42010-03-08 13:35:02 +01004305 obj_priv = to_intel_bo(obj);
Chris Wilson3ef94da2009-09-14 16:50:29 +01004306
4307 if (obj_priv->pin_count) {
4308 drm_gem_object_unreference(obj);
4309 mutex_unlock(&dev->struct_mutex);
4310
4311 DRM_ERROR("Attempted i915_gem_madvise_ioctl() on a pinned object\n");
4312 return -EINVAL;
4313 }
4314
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004315 if (obj_priv->madv != __I915_MADV_PURGED)
4316 obj_priv->madv = args->madv;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004317
Chris Wilson2d7ef392009-09-20 23:13:10 +01004318 /* if the object is no longer bound, discard its backing storage */
4319 if (i915_gem_object_is_purgeable(obj_priv) &&
4320 obj_priv->gtt_space == NULL)
4321 i915_gem_object_truncate(obj);
4322
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004323 args->retained = obj_priv->madv != __I915_MADV_PURGED;
4324
Chris Wilson3ef94da2009-09-14 16:50:29 +01004325 drm_gem_object_unreference(obj);
4326 mutex_unlock(&dev->struct_mutex);
4327
4328 return 0;
4329}
4330
Daniel Vetterac52bc52010-04-09 19:05:06 +00004331struct drm_gem_object * i915_gem_alloc_object(struct drm_device *dev,
4332 size_t size)
4333{
Daniel Vetterc397b902010-04-09 19:05:07 +00004334 struct drm_i915_gem_object *obj;
4335
4336 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
4337 if (obj == NULL)
4338 return NULL;
4339
4340 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4341 kfree(obj);
4342 return NULL;
4343 }
4344
4345 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4346 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4347
4348 obj->agp_type = AGP_USER_MEMORY;
Daniel Vetter62b8b212010-04-09 19:05:08 +00004349 obj->base.driver_private = NULL;
Daniel Vetterc397b902010-04-09 19:05:07 +00004350 obj->fence_reg = I915_FENCE_REG_NONE;
4351 INIT_LIST_HEAD(&obj->list);
4352 INIT_LIST_HEAD(&obj->gpu_write_list);
Daniel Vetterc397b902010-04-09 19:05:07 +00004353 obj->madv = I915_MADV_WILLNEED;
4354
4355 trace_i915_gem_object_create(&obj->base);
4356
4357 return &obj->base;
Daniel Vetterac52bc52010-04-09 19:05:06 +00004358}
4359
Eric Anholt673a3942008-07-30 12:06:12 -07004360int i915_gem_init_object(struct drm_gem_object *obj)
4361{
Daniel Vetterc397b902010-04-09 19:05:07 +00004362 BUG();
Jesse Barnesde151cf2008-11-12 10:03:55 -08004363
Eric Anholt673a3942008-07-30 12:06:12 -07004364 return 0;
4365}
4366
Chris Wilsonbe726152010-07-23 23:18:50 +01004367static void i915_gem_free_object_tail(struct drm_gem_object *obj)
4368{
4369 struct drm_device *dev = obj->dev;
4370 drm_i915_private_t *dev_priv = dev->dev_private;
4371 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
4372 int ret;
4373
4374 ret = i915_gem_object_unbind(obj);
4375 if (ret == -ERESTARTSYS) {
4376 list_move(&obj_priv->list,
4377 &dev_priv->mm.deferred_free_list);
4378 return;
4379 }
4380
4381 if (obj_priv->mmap_offset)
4382 i915_gem_free_mmap_offset(obj);
4383
4384 drm_gem_object_release(obj);
4385
4386 kfree(obj_priv->page_cpu_valid);
4387 kfree(obj_priv->bit_17);
4388 kfree(obj_priv);
4389}
4390
Eric Anholt673a3942008-07-30 12:06:12 -07004391void i915_gem_free_object(struct drm_gem_object *obj)
4392{
Jesse Barnesde151cf2008-11-12 10:03:55 -08004393 struct drm_device *dev = obj->dev;
Daniel Vetter23010e42010-03-08 13:35:02 +01004394 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004395
Chris Wilson1c5d22f2009-08-25 11:15:50 +01004396 trace_i915_gem_object_destroy(obj);
4397
Eric Anholt673a3942008-07-30 12:06:12 -07004398 while (obj_priv->pin_count > 0)
4399 i915_gem_object_unpin(obj);
4400
Dave Airlie71acb5e2008-12-30 20:31:46 +10004401 if (obj_priv->phys_obj)
4402 i915_gem_detach_phys_object(dev, obj);
4403
Chris Wilsonbe726152010-07-23 23:18:50 +01004404 i915_gem_free_object_tail(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07004405}
4406
Jesse Barnes5669fca2009-02-17 15:13:31 -08004407int
Eric Anholt673a3942008-07-30 12:06:12 -07004408i915_gem_idle(struct drm_device *dev)
4409{
4410 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson29105cc2010-01-07 10:39:13 +00004411 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004412
Keith Packard6dbe2772008-10-14 21:41:13 -07004413 mutex_lock(&dev->struct_mutex);
4414
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004415 if (dev_priv->mm.suspended ||
Zou Nan haid1b851f2010-05-21 09:08:57 +08004416 (dev_priv->render_ring.gem_object == NULL) ||
4417 (HAS_BSD(dev) &&
4418 dev_priv->bsd_ring.gem_object == NULL)) {
Keith Packard6dbe2772008-10-14 21:41:13 -07004419 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07004420 return 0;
Keith Packard6dbe2772008-10-14 21:41:13 -07004421 }
Eric Anholt673a3942008-07-30 12:06:12 -07004422
Chris Wilson29105cc2010-01-07 10:39:13 +00004423 ret = i915_gpu_idle(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07004424 if (ret) {
4425 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07004426 return ret;
Keith Packard6dbe2772008-10-14 21:41:13 -07004427 }
Eric Anholt673a3942008-07-30 12:06:12 -07004428
Chris Wilson29105cc2010-01-07 10:39:13 +00004429 /* Under UMS, be paranoid and evict. */
4430 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
Chris Wilsonb47eb4a2010-08-07 11:01:23 +01004431 ret = i915_gem_evict_inactive(dev);
Chris Wilson29105cc2010-01-07 10:39:13 +00004432 if (ret) {
4433 mutex_unlock(&dev->struct_mutex);
4434 return ret;
4435 }
4436 }
4437
4438 /* Hack! Don't let anybody do execbuf while we don't control the chip.
4439 * We need to replace this with a semaphore, or something.
4440 * And not confound mm.suspended!
4441 */
4442 dev_priv->mm.suspended = 1;
Daniel Vetterbc0c7f12010-08-20 18:18:48 +02004443 del_timer_sync(&dev_priv->hangcheck_timer);
Chris Wilson29105cc2010-01-07 10:39:13 +00004444
4445 i915_kernel_lost_context(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -07004446 i915_gem_cleanup_ringbuffer(dev);
Chris Wilson29105cc2010-01-07 10:39:13 +00004447
Keith Packard6dbe2772008-10-14 21:41:13 -07004448 mutex_unlock(&dev->struct_mutex);
4449
Chris Wilson29105cc2010-01-07 10:39:13 +00004450 /* Cancel the retire work handler, which should be idle now. */
4451 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4452
Eric Anholt673a3942008-07-30 12:06:12 -07004453 return 0;
4454}
4455
Jesse Barnese552eb72010-04-21 11:39:23 -07004456/*
4457 * 965+ support PIPE_CONTROL commands, which provide finer grained control
4458 * over cache flushing.
4459 */
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004460static int
Jesse Barnese552eb72010-04-21 11:39:23 -07004461i915_gem_init_pipe_control(struct drm_device *dev)
4462{
4463 drm_i915_private_t *dev_priv = dev->dev_private;
4464 struct drm_gem_object *obj;
4465 struct drm_i915_gem_object *obj_priv;
4466 int ret;
4467
Eric Anholt34dc4d42010-05-07 14:30:03 -07004468 obj = i915_gem_alloc_object(dev, 4096);
Jesse Barnese552eb72010-04-21 11:39:23 -07004469 if (obj == NULL) {
4470 DRM_ERROR("Failed to allocate seqno page\n");
4471 ret = -ENOMEM;
4472 goto err;
4473 }
4474 obj_priv = to_intel_bo(obj);
4475 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
4476
4477 ret = i915_gem_object_pin(obj, 4096);
4478 if (ret)
4479 goto err_unref;
4480
4481 dev_priv->seqno_gfx_addr = obj_priv->gtt_offset;
4482 dev_priv->seqno_page = kmap(obj_priv->pages[0]);
4483 if (dev_priv->seqno_page == NULL)
4484 goto err_unpin;
4485
4486 dev_priv->seqno_obj = obj;
4487 memset(dev_priv->seqno_page, 0, PAGE_SIZE);
4488
4489 return 0;
4490
4491err_unpin:
4492 i915_gem_object_unpin(obj);
4493err_unref:
4494 drm_gem_object_unreference(obj);
4495err:
4496 return ret;
4497}
4498
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004499
4500static void
Jesse Barnese552eb72010-04-21 11:39:23 -07004501i915_gem_cleanup_pipe_control(struct drm_device *dev)
4502{
4503 drm_i915_private_t *dev_priv = dev->dev_private;
4504 struct drm_gem_object *obj;
4505 struct drm_i915_gem_object *obj_priv;
4506
4507 obj = dev_priv->seqno_obj;
4508 obj_priv = to_intel_bo(obj);
4509 kunmap(obj_priv->pages[0]);
4510 i915_gem_object_unpin(obj);
4511 drm_gem_object_unreference(obj);
4512 dev_priv->seqno_obj = NULL;
4513
4514 dev_priv->seqno_page = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07004515}
4516
Eric Anholt673a3942008-07-30 12:06:12 -07004517int
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004518i915_gem_init_ringbuffer(struct drm_device *dev)
4519{
4520 drm_i915_private_t *dev_priv = dev->dev_private;
4521 int ret;
Chris Wilson68f95ba2010-05-27 13:18:22 +01004522
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004523 if (HAS_PIPE_CONTROL(dev)) {
4524 ret = i915_gem_init_pipe_control(dev);
4525 if (ret)
4526 return ret;
4527 }
Chris Wilson68f95ba2010-05-27 13:18:22 +01004528
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08004529 ret = intel_init_render_ring_buffer(dev);
Chris Wilson68f95ba2010-05-27 13:18:22 +01004530 if (ret)
4531 goto cleanup_pipe_control;
4532
4533 if (HAS_BSD(dev)) {
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08004534 ret = intel_init_bsd_ring_buffer(dev);
Chris Wilson68f95ba2010-05-27 13:18:22 +01004535 if (ret)
4536 goto cleanup_render_ring;
Zou Nan haid1b851f2010-05-21 09:08:57 +08004537 }
Chris Wilson68f95ba2010-05-27 13:18:22 +01004538
Chris Wilson6f392d52010-08-07 11:01:22 +01004539 dev_priv->next_seqno = 1;
4540
Chris Wilson68f95ba2010-05-27 13:18:22 +01004541 return 0;
4542
4543cleanup_render_ring:
4544 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
4545cleanup_pipe_control:
4546 if (HAS_PIPE_CONTROL(dev))
4547 i915_gem_cleanup_pipe_control(dev);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004548 return ret;
4549}
4550
4551void
4552i915_gem_cleanup_ringbuffer(struct drm_device *dev)
4553{
4554 drm_i915_private_t *dev_priv = dev->dev_private;
4555
4556 intel_cleanup_ring_buffer(dev, &dev_priv->render_ring);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004557 if (HAS_BSD(dev))
4558 intel_cleanup_ring_buffer(dev, &dev_priv->bsd_ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08004559 if (HAS_PIPE_CONTROL(dev))
4560 i915_gem_cleanup_pipe_control(dev);
4561}
4562
4563int
Eric Anholt673a3942008-07-30 12:06:12 -07004564i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
4565 struct drm_file *file_priv)
4566{
4567 drm_i915_private_t *dev_priv = dev->dev_private;
4568 int ret;
4569
Jesse Barnes79e53942008-11-07 14:24:08 -08004570 if (drm_core_check_feature(dev, DRIVER_MODESET))
4571 return 0;
4572
Ben Gamariba1234d2009-09-14 17:48:47 -04004573 if (atomic_read(&dev_priv->mm.wedged)) {
Eric Anholt673a3942008-07-30 12:06:12 -07004574 DRM_ERROR("Reenabling wedged hardware, good luck\n");
Ben Gamariba1234d2009-09-14 17:48:47 -04004575 atomic_set(&dev_priv->mm.wedged, 0);
Eric Anholt673a3942008-07-30 12:06:12 -07004576 }
4577
Eric Anholt673a3942008-07-30 12:06:12 -07004578 mutex_lock(&dev->struct_mutex);
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004579 dev_priv->mm.suspended = 0;
4580
4581 ret = i915_gem_init_ringbuffer(dev);
Wu Fengguangd816f6a2009-04-18 10:43:32 +08004582 if (ret != 0) {
4583 mutex_unlock(&dev->struct_mutex);
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004584 return ret;
Wu Fengguangd816f6a2009-04-18 10:43:32 +08004585 }
Eric Anholt9bb2d6f2008-12-23 18:42:32 -08004586
Zou Nan hai852835f2010-05-21 09:08:56 +08004587 BUG_ON(!list_empty(&dev_priv->render_ring.active_list));
Zou Nan haid1b851f2010-05-21 09:08:57 +08004588 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.active_list));
Eric Anholt673a3942008-07-30 12:06:12 -07004589 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
4590 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
Zou Nan hai852835f2010-05-21 09:08:56 +08004591 BUG_ON(!list_empty(&dev_priv->render_ring.request_list));
Zou Nan haid1b851f2010-05-21 09:08:57 +08004592 BUG_ON(HAS_BSD(dev) && !list_empty(&dev_priv->bsd_ring.request_list));
Eric Anholt673a3942008-07-30 12:06:12 -07004593 mutex_unlock(&dev->struct_mutex);
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004594
Chris Wilson5f353082010-06-07 14:03:03 +01004595 ret = drm_irq_install(dev);
4596 if (ret)
4597 goto cleanup_ringbuffer;
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004598
Eric Anholt673a3942008-07-30 12:06:12 -07004599 return 0;
Chris Wilson5f353082010-06-07 14:03:03 +01004600
4601cleanup_ringbuffer:
4602 mutex_lock(&dev->struct_mutex);
4603 i915_gem_cleanup_ringbuffer(dev);
4604 dev_priv->mm.suspended = 1;
4605 mutex_unlock(&dev->struct_mutex);
4606
4607 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004608}
4609
4610int
4611i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
4612 struct drm_file *file_priv)
4613{
Jesse Barnes79e53942008-11-07 14:24:08 -08004614 if (drm_core_check_feature(dev, DRIVER_MODESET))
4615 return 0;
4616
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04004617 drm_irq_uninstall(dev);
Linus Torvaldse6890f62009-09-08 17:09:24 -07004618 return i915_gem_idle(dev);
Eric Anholt673a3942008-07-30 12:06:12 -07004619}
4620
4621void
4622i915_gem_lastclose(struct drm_device *dev)
4623{
4624 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004625
Eric Anholte806b492009-01-22 09:56:58 -08004626 if (drm_core_check_feature(dev, DRIVER_MODESET))
4627 return;
4628
Keith Packard6dbe2772008-10-14 21:41:13 -07004629 ret = i915_gem_idle(dev);
4630 if (ret)
4631 DRM_ERROR("failed to idle hardware: %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07004632}
4633
4634void
4635i915_gem_load(struct drm_device *dev)
4636{
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004637 int i;
Eric Anholt673a3942008-07-30 12:06:12 -07004638 drm_i915_private_t *dev_priv = dev->dev_private;
4639
Eric Anholt673a3942008-07-30 12:06:12 -07004640 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
Daniel Vetter99fcb762010-02-07 16:20:18 +01004641 INIT_LIST_HEAD(&dev_priv->mm.gpu_write_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004642 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
Chris Wilsonf13d3f72010-09-20 17:36:15 +01004643 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
Eric Anholta09ba7f2009-08-29 12:49:51 -07004644 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
Chris Wilsonbe726152010-07-23 23:18:50 +01004645 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
Zou Nan hai852835f2010-05-21 09:08:56 +08004646 INIT_LIST_HEAD(&dev_priv->render_ring.active_list);
4647 INIT_LIST_HEAD(&dev_priv->render_ring.request_list);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004648 if (HAS_BSD(dev)) {
4649 INIT_LIST_HEAD(&dev_priv->bsd_ring.active_list);
4650 INIT_LIST_HEAD(&dev_priv->bsd_ring.request_list);
4651 }
Daniel Vetter007cc8a2010-04-28 11:02:31 +02004652 for (i = 0; i < 16; i++)
4653 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
Eric Anholt673a3942008-07-30 12:06:12 -07004654 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
4655 i915_gem_retire_work_handler);
Chris Wilson31169712009-09-14 16:50:28 +01004656 spin_lock(&shrink_list_lock);
4657 list_add(&dev_priv->mm.shrink_list, &shrink_list);
4658 spin_unlock(&shrink_list_lock);
4659
Dave Airlie94400122010-07-20 13:15:31 +10004660 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
4661 if (IS_GEN3(dev)) {
4662 u32 tmp = I915_READ(MI_ARB_STATE);
4663 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
4664 /* arb state is a masked write, so set bit + bit in mask */
4665 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
4666 I915_WRITE(MI_ARB_STATE, tmp);
4667 }
4668 }
4669
Jesse Barnesde151cf2008-11-12 10:03:55 -08004670 /* Old X drivers will take 0-2 for front, back, depth buffers */
Eric Anholtb397c832010-01-26 09:43:10 -08004671 if (!drm_core_check_feature(dev, DRIVER_MODESET))
4672 dev_priv->fence_reg_start = 3;
Jesse Barnesde151cf2008-11-12 10:03:55 -08004673
Chris Wilsona6c45cf2010-09-17 00:32:17 +01004674 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
Jesse Barnesde151cf2008-11-12 10:03:55 -08004675 dev_priv->num_fence_regs = 16;
4676 else
4677 dev_priv->num_fence_regs = 8;
4678
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004679 /* Initialize fence registers to zero */
Chris Wilsona6c45cf2010-09-17 00:32:17 +01004680 switch (INTEL_INFO(dev)->gen) {
4681 case 6:
4682 for (i = 0; i < 16; i++)
4683 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + (i * 8), 0);
4684 break;
4685 case 5:
4686 case 4:
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004687 for (i = 0; i < 16; i++)
4688 I915_WRITE64(FENCE_REG_965_0 + (i * 8), 0);
Chris Wilsona6c45cf2010-09-17 00:32:17 +01004689 break;
4690 case 3:
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004691 if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
4692 for (i = 0; i < 8; i++)
4693 I915_WRITE(FENCE_REG_945_8 + (i * 4), 0);
Chris Wilsona6c45cf2010-09-17 00:32:17 +01004694 case 2:
4695 for (i = 0; i < 8; i++)
4696 I915_WRITE(FENCE_REG_830_0 + (i * 4), 0);
4697 break;
Grégoire Henryb5aa8a02009-06-23 15:41:02 +02004698 }
Eric Anholt673a3942008-07-30 12:06:12 -07004699 i915_gem_detect_bit_6_swizzle(dev);
Kristian Høgsberg6b95a202009-11-18 11:25:18 -05004700 init_waitqueue_head(&dev_priv->pending_flip_queue);
Eric Anholt673a3942008-07-30 12:06:12 -07004701}
Dave Airlie71acb5e2008-12-30 20:31:46 +10004702
4703/*
4704 * Create a physically contiguous memory object for this object
4705 * e.g. for cursor + overlay regs
4706 */
Chris Wilson995b6762010-08-20 13:23:26 +01004707static int i915_gem_init_phys_object(struct drm_device *dev,
4708 int id, int size, int align)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004709{
4710 drm_i915_private_t *dev_priv = dev->dev_private;
4711 struct drm_i915_gem_phys_object *phys_obj;
4712 int ret;
4713
4714 if (dev_priv->mm.phys_objs[id - 1] || !size)
4715 return 0;
4716
Eric Anholt9a298b22009-03-24 12:23:04 -07004717 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004718 if (!phys_obj)
4719 return -ENOMEM;
4720
4721 phys_obj->id = id;
4722
Chris Wilson6eeefaf2010-08-07 11:01:39 +01004723 phys_obj->handle = drm_pci_alloc(dev, size, align);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004724 if (!phys_obj->handle) {
4725 ret = -ENOMEM;
4726 goto kfree_obj;
4727 }
4728#ifdef CONFIG_X86
4729 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4730#endif
4731
4732 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4733
4734 return 0;
4735kfree_obj:
Eric Anholt9a298b22009-03-24 12:23:04 -07004736 kfree(phys_obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004737 return ret;
4738}
4739
Chris Wilson995b6762010-08-20 13:23:26 +01004740static void i915_gem_free_phys_object(struct drm_device *dev, int id)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004741{
4742 drm_i915_private_t *dev_priv = dev->dev_private;
4743 struct drm_i915_gem_phys_object *phys_obj;
4744
4745 if (!dev_priv->mm.phys_objs[id - 1])
4746 return;
4747
4748 phys_obj = dev_priv->mm.phys_objs[id - 1];
4749 if (phys_obj->cur_obj) {
4750 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4751 }
4752
4753#ifdef CONFIG_X86
4754 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4755#endif
4756 drm_pci_free(dev, phys_obj->handle);
4757 kfree(phys_obj);
4758 dev_priv->mm.phys_objs[id - 1] = NULL;
4759}
4760
4761void i915_gem_free_all_phys_object(struct drm_device *dev)
4762{
4763 int i;
4764
Dave Airlie260883c2009-01-22 17:58:49 +10004765 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004766 i915_gem_free_phys_object(dev, i);
4767}
4768
4769void i915_gem_detach_phys_object(struct drm_device *dev,
4770 struct drm_gem_object *obj)
4771{
4772 struct drm_i915_gem_object *obj_priv;
4773 int i;
4774 int ret;
4775 int page_count;
4776
Daniel Vetter23010e42010-03-08 13:35:02 +01004777 obj_priv = to_intel_bo(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004778 if (!obj_priv->phys_obj)
4779 return;
4780
Chris Wilson4bdadb92010-01-27 13:36:32 +00004781 ret = i915_gem_object_get_pages(obj, 0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004782 if (ret)
4783 goto out;
4784
4785 page_count = obj->size / PAGE_SIZE;
4786
4787 for (i = 0; i < page_count; i++) {
Eric Anholt856fa192009-03-19 14:10:50 -07004788 char *dst = kmap_atomic(obj_priv->pages[i], KM_USER0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004789 char *src = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4790
4791 memcpy(dst, src, PAGE_SIZE);
4792 kunmap_atomic(dst, KM_USER0);
4793 }
Eric Anholt856fa192009-03-19 14:10:50 -07004794 drm_clflush_pages(obj_priv->pages, page_count);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004795 drm_agp_chipset_flush(dev);
Chris Wilsond78b47b2009-06-17 21:52:49 +01004796
4797 i915_gem_object_put_pages(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004798out:
4799 obj_priv->phys_obj->cur_obj = NULL;
4800 obj_priv->phys_obj = NULL;
4801}
4802
4803int
4804i915_gem_attach_phys_object(struct drm_device *dev,
Chris Wilson6eeefaf2010-08-07 11:01:39 +01004805 struct drm_gem_object *obj,
4806 int id,
4807 int align)
Dave Airlie71acb5e2008-12-30 20:31:46 +10004808{
4809 drm_i915_private_t *dev_priv = dev->dev_private;
4810 struct drm_i915_gem_object *obj_priv;
4811 int ret = 0;
4812 int page_count;
4813 int i;
4814
4815 if (id > I915_MAX_PHYS_OBJECT)
4816 return -EINVAL;
4817
Daniel Vetter23010e42010-03-08 13:35:02 +01004818 obj_priv = to_intel_bo(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004819
4820 if (obj_priv->phys_obj) {
4821 if (obj_priv->phys_obj->id == id)
4822 return 0;
4823 i915_gem_detach_phys_object(dev, obj);
4824 }
4825
Dave Airlie71acb5e2008-12-30 20:31:46 +10004826 /* create a new object */
4827 if (!dev_priv->mm.phys_objs[id - 1]) {
4828 ret = i915_gem_init_phys_object(dev, id,
Chris Wilson6eeefaf2010-08-07 11:01:39 +01004829 obj->size, align);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004830 if (ret) {
Linus Torvaldsaeb565d2009-01-26 10:01:53 -08004831 DRM_ERROR("failed to init phys object %d size: %zu\n", id, obj->size);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004832 goto out;
4833 }
4834 }
4835
4836 /* bind to the object */
4837 obj_priv->phys_obj = dev_priv->mm.phys_objs[id - 1];
4838 obj_priv->phys_obj->cur_obj = obj;
4839
Chris Wilson4bdadb92010-01-27 13:36:32 +00004840 ret = i915_gem_object_get_pages(obj, 0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004841 if (ret) {
4842 DRM_ERROR("failed to get page list\n");
4843 goto out;
4844 }
4845
4846 page_count = obj->size / PAGE_SIZE;
4847
4848 for (i = 0; i < page_count; i++) {
Eric Anholt856fa192009-03-19 14:10:50 -07004849 char *src = kmap_atomic(obj_priv->pages[i], KM_USER0);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004850 char *dst = obj_priv->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4851
4852 memcpy(dst, src, PAGE_SIZE);
4853 kunmap_atomic(src, KM_USER0);
4854 }
4855
Chris Wilsond78b47b2009-06-17 21:52:49 +01004856 i915_gem_object_put_pages(obj);
4857
Dave Airlie71acb5e2008-12-30 20:31:46 +10004858 return 0;
4859out:
4860 return ret;
4861}
4862
4863static int
4864i915_gem_phys_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
4865 struct drm_i915_gem_pwrite *args,
4866 struct drm_file *file_priv)
4867{
Daniel Vetter23010e42010-03-08 13:35:02 +01004868 struct drm_i915_gem_object *obj_priv = to_intel_bo(obj);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004869 void *obj_addr;
4870 int ret;
4871 char __user *user_data;
4872
4873 user_data = (char __user *) (uintptr_t) args->data_ptr;
4874 obj_addr = obj_priv->phys_obj->handle->vaddr + args->offset;
4875
Zhao Yakui44d98a62009-10-09 11:39:40 +08004876 DRM_DEBUG_DRIVER("obj_addr %p, %lld\n", obj_addr, args->size);
Dave Airlie71acb5e2008-12-30 20:31:46 +10004877 ret = copy_from_user(obj_addr, user_data, args->size);
4878 if (ret)
4879 return -EFAULT;
4880
4881 drm_agp_chipset_flush(dev);
4882 return 0;
4883}
Eric Anholtb9624422009-06-03 07:27:35 +00004884
4885void i915_gem_release(struct drm_device * dev, struct drm_file *file_priv)
4886{
4887 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
4888
4889 /* Clean up our request list when the client is going away, so that
4890 * later retire_requests won't dereference our soon-to-be-gone
4891 * file_priv.
4892 */
4893 mutex_lock(&dev->struct_mutex);
4894 while (!list_empty(&i915_file_priv->mm.request_list))
4895 list_del_init(i915_file_priv->mm.request_list.next);
4896 mutex_unlock(&dev->struct_mutex);
4897}
Chris Wilson31169712009-09-14 16:50:28 +01004898
Chris Wilson31169712009-09-14 16:50:28 +01004899static int
Chris Wilson1637ef42010-04-20 17:10:35 +01004900i915_gpu_is_active(struct drm_device *dev)
4901{
4902 drm_i915_private_t *dev_priv = dev->dev_private;
4903 int lists_empty;
4904
Chris Wilson1637ef42010-04-20 17:10:35 +01004905 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
Zou Nan hai852835f2010-05-21 09:08:56 +08004906 list_empty(&dev_priv->render_ring.active_list);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004907 if (HAS_BSD(dev))
4908 lists_empty &= list_empty(&dev_priv->bsd_ring.active_list);
Chris Wilson1637ef42010-04-20 17:10:35 +01004909
4910 return !lists_empty;
4911}
4912
4913static int
Dave Chinner7f8275d2010-07-19 14:56:17 +10004914i915_gem_shrink(struct shrinker *shrink, int nr_to_scan, gfp_t gfp_mask)
Chris Wilson31169712009-09-14 16:50:28 +01004915{
4916 drm_i915_private_t *dev_priv, *next_dev;
4917 struct drm_i915_gem_object *obj_priv, *next_obj;
4918 int cnt = 0;
4919 int would_deadlock = 1;
4920
4921 /* "fast-path" to count number of available objects */
4922 if (nr_to_scan == 0) {
4923 spin_lock(&shrink_list_lock);
4924 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
4925 struct drm_device *dev = dev_priv->dev;
4926
4927 if (mutex_trylock(&dev->struct_mutex)) {
4928 list_for_each_entry(obj_priv,
4929 &dev_priv->mm.inactive_list,
4930 list)
4931 cnt++;
4932 mutex_unlock(&dev->struct_mutex);
4933 }
4934 }
4935 spin_unlock(&shrink_list_lock);
4936
4937 return (cnt / 100) * sysctl_vfs_cache_pressure;
4938 }
4939
4940 spin_lock(&shrink_list_lock);
4941
Chris Wilson1637ef42010-04-20 17:10:35 +01004942rescan:
Chris Wilson31169712009-09-14 16:50:28 +01004943 /* first scan for clean buffers */
4944 list_for_each_entry_safe(dev_priv, next_dev,
4945 &shrink_list, mm.shrink_list) {
4946 struct drm_device *dev = dev_priv->dev;
4947
4948 if (! mutex_trylock(&dev->struct_mutex))
4949 continue;
4950
4951 spin_unlock(&shrink_list_lock);
Chris Wilsonb09a1fe2010-07-23 23:18:49 +01004952 i915_gem_retire_requests(dev);
Zou Nan haid1b851f2010-05-21 09:08:57 +08004953
Chris Wilson31169712009-09-14 16:50:28 +01004954 list_for_each_entry_safe(obj_priv, next_obj,
4955 &dev_priv->mm.inactive_list,
4956 list) {
4957 if (i915_gem_object_is_purgeable(obj_priv)) {
Daniel Vettera8089e82010-04-09 19:05:09 +00004958 i915_gem_object_unbind(&obj_priv->base);
Chris Wilson31169712009-09-14 16:50:28 +01004959 if (--nr_to_scan <= 0)
4960 break;
4961 }
4962 }
4963
4964 spin_lock(&shrink_list_lock);
4965 mutex_unlock(&dev->struct_mutex);
4966
Chris Wilson963b4832009-09-20 23:03:54 +01004967 would_deadlock = 0;
4968
Chris Wilson31169712009-09-14 16:50:28 +01004969 if (nr_to_scan <= 0)
4970 break;
4971 }
4972
4973 /* second pass, evict/count anything still on the inactive list */
4974 list_for_each_entry_safe(dev_priv, next_dev,
4975 &shrink_list, mm.shrink_list) {
4976 struct drm_device *dev = dev_priv->dev;
4977
4978 if (! mutex_trylock(&dev->struct_mutex))
4979 continue;
4980
4981 spin_unlock(&shrink_list_lock);
4982
4983 list_for_each_entry_safe(obj_priv, next_obj,
4984 &dev_priv->mm.inactive_list,
4985 list) {
4986 if (nr_to_scan > 0) {
Daniel Vettera8089e82010-04-09 19:05:09 +00004987 i915_gem_object_unbind(&obj_priv->base);
Chris Wilson31169712009-09-14 16:50:28 +01004988 nr_to_scan--;
4989 } else
4990 cnt++;
4991 }
4992
4993 spin_lock(&shrink_list_lock);
4994 mutex_unlock(&dev->struct_mutex);
4995
4996 would_deadlock = 0;
4997 }
4998
Chris Wilson1637ef42010-04-20 17:10:35 +01004999 if (nr_to_scan) {
5000 int active = 0;
5001
5002 /*
5003 * We are desperate for pages, so as a last resort, wait
5004 * for the GPU to finish and discard whatever we can.
5005 * This has a dramatic impact to reduce the number of
5006 * OOM-killer events whilst running the GPU aggressively.
5007 */
5008 list_for_each_entry(dev_priv, &shrink_list, mm.shrink_list) {
5009 struct drm_device *dev = dev_priv->dev;
5010
5011 if (!mutex_trylock(&dev->struct_mutex))
5012 continue;
5013
5014 spin_unlock(&shrink_list_lock);
5015
5016 if (i915_gpu_is_active(dev)) {
5017 i915_gpu_idle(dev);
5018 active++;
5019 }
5020
5021 spin_lock(&shrink_list_lock);
5022 mutex_unlock(&dev->struct_mutex);
5023 }
5024
5025 if (active)
5026 goto rescan;
5027 }
5028
Chris Wilson31169712009-09-14 16:50:28 +01005029 spin_unlock(&shrink_list_lock);
5030
5031 if (would_deadlock)
5032 return -1;
5033 else if (cnt > 0)
5034 return (cnt / 100) * sysctl_vfs_cache_pressure;
5035 else
5036 return 0;
5037}
5038
5039static struct shrinker shrinker = {
5040 .shrink = i915_gem_shrink,
5041 .seeks = DEFAULT_SEEKS,
5042};
5043
5044__init void
5045i915_gem_shrinker_init(void)
5046{
5047 register_shrinker(&shrinker);
5048}
5049
5050__exit void
5051i915_gem_shrinker_exit(void)
5052{
5053 unregister_shrinker(&shrinker);
5054}