blob: 174c0c3ba0b0f25079c9ae39db250fdbfe256612 [file] [log] [blame]
Eric Anholt673a3942008-07-30 12:06:12 -07001/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
32#include <linux/swap.h>
33
Eric Anholt28dfe522008-11-13 15:00:55 -080034#define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
35
Keith Packardc0d90822008-11-20 23:11:08 -080036static void
37i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
38 uint32_t read_domains,
39 uint32_t write_domain);
Eric Anholte47c68e2008-11-14 13:35:19 -080040static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
41static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
42static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
Eric Anholt2ef7eea2008-11-10 10:53:25 -080043static int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj,
44 int write);
Eric Anholte47c68e2008-11-14 13:35:19 -080045static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
46 int write);
47static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
48 uint64_t offset,
49 uint64_t size);
50static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
Eric Anholt673a3942008-07-30 12:06:12 -070051static int i915_gem_object_get_page_list(struct drm_gem_object *obj);
52static void i915_gem_object_free_page_list(struct drm_gem_object *obj);
53static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
54
Keith Packard6dbe2772008-10-14 21:41:13 -070055static void
56i915_gem_cleanup_ringbuffer(struct drm_device *dev);
57
Eric Anholt673a3942008-07-30 12:06:12 -070058int
59i915_gem_init_ioctl(struct drm_device *dev, void *data,
60 struct drm_file *file_priv)
61{
62 drm_i915_private_t *dev_priv = dev->dev_private;
63 struct drm_i915_gem_init *args = data;
64
65 mutex_lock(&dev->struct_mutex);
66
67 if (args->gtt_start >= args->gtt_end ||
68 (args->gtt_start & (PAGE_SIZE - 1)) != 0 ||
69 (args->gtt_end & (PAGE_SIZE - 1)) != 0) {
70 mutex_unlock(&dev->struct_mutex);
71 return -EINVAL;
72 }
73
74 drm_mm_init(&dev_priv->mm.gtt_space, args->gtt_start,
75 args->gtt_end - args->gtt_start);
76
77 dev->gtt_total = (uint32_t) (args->gtt_end - args->gtt_start);
78
79 mutex_unlock(&dev->struct_mutex);
80
81 return 0;
82}
83
Eric Anholt5a125c32008-10-22 21:40:13 -070084int
85i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
86 struct drm_file *file_priv)
87{
Eric Anholt5a125c32008-10-22 21:40:13 -070088 struct drm_i915_gem_get_aperture *args = data;
Eric Anholt5a125c32008-10-22 21:40:13 -070089
90 if (!(dev->driver->driver_features & DRIVER_GEM))
91 return -ENODEV;
92
93 args->aper_size = dev->gtt_total;
Keith Packard2678d9d2008-11-20 22:54:54 -080094 args->aper_available_size = (args->aper_size -
95 atomic_read(&dev->pin_memory));
Eric Anholt5a125c32008-10-22 21:40:13 -070096
97 return 0;
98}
99
Eric Anholt673a3942008-07-30 12:06:12 -0700100
101/**
102 * Creates a new mm object and returns a handle to it.
103 */
104int
105i915_gem_create_ioctl(struct drm_device *dev, void *data,
106 struct drm_file *file_priv)
107{
108 struct drm_i915_gem_create *args = data;
109 struct drm_gem_object *obj;
110 int handle, ret;
111
112 args->size = roundup(args->size, PAGE_SIZE);
113
114 /* Allocate the new object */
115 obj = drm_gem_object_alloc(dev, args->size);
116 if (obj == NULL)
117 return -ENOMEM;
118
119 ret = drm_gem_handle_create(file_priv, obj, &handle);
120 mutex_lock(&dev->struct_mutex);
121 drm_gem_object_handle_unreference(obj);
122 mutex_unlock(&dev->struct_mutex);
123
124 if (ret)
125 return ret;
126
127 args->handle = handle;
128
129 return 0;
130}
131
132/**
133 * Reads data from the object referenced by handle.
134 *
135 * On error, the contents of *data are undefined.
136 */
137int
138i915_gem_pread_ioctl(struct drm_device *dev, void *data,
139 struct drm_file *file_priv)
140{
141 struct drm_i915_gem_pread *args = data;
142 struct drm_gem_object *obj;
143 struct drm_i915_gem_object *obj_priv;
144 ssize_t read;
145 loff_t offset;
146 int ret;
147
148 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
149 if (obj == NULL)
150 return -EBADF;
151 obj_priv = obj->driver_private;
152
153 /* Bounds check source.
154 *
155 * XXX: This could use review for overflow issues...
156 */
157 if (args->offset > obj->size || args->size > obj->size ||
158 args->offset + args->size > obj->size) {
159 drm_gem_object_unreference(obj);
160 return -EINVAL;
161 }
162
163 mutex_lock(&dev->struct_mutex);
164
Eric Anholte47c68e2008-11-14 13:35:19 -0800165 ret = i915_gem_object_set_cpu_read_domain_range(obj, args->offset,
166 args->size);
Eric Anholt673a3942008-07-30 12:06:12 -0700167 if (ret != 0) {
168 drm_gem_object_unreference(obj);
169 mutex_unlock(&dev->struct_mutex);
Dave Airliee7d22bc2008-10-07 13:40:36 +1000170 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700171 }
172
173 offset = args->offset;
174
175 read = vfs_read(obj->filp, (char __user *)(uintptr_t)args->data_ptr,
176 args->size, &offset);
177 if (read != args->size) {
178 drm_gem_object_unreference(obj);
179 mutex_unlock(&dev->struct_mutex);
180 if (read < 0)
181 return read;
182 else
183 return -EINVAL;
184 }
185
186 drm_gem_object_unreference(obj);
187 mutex_unlock(&dev->struct_mutex);
188
189 return 0;
190}
191
Keith Packard0839ccb2008-10-30 19:38:48 -0700192/* This is the fast write path which cannot handle
193 * page faults in the source data
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700194 */
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700195
Keith Packard0839ccb2008-10-30 19:38:48 -0700196static inline int
197fast_user_write(struct io_mapping *mapping,
198 loff_t page_base, int page_offset,
199 char __user *user_data,
200 int length)
201{
202 char *vaddr_atomic;
203 unsigned long unwritten;
204
205 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
206 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
207 user_data, length);
208 io_mapping_unmap_atomic(vaddr_atomic);
209 if (unwritten)
210 return -EFAULT;
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700211 return 0;
Keith Packard0839ccb2008-10-30 19:38:48 -0700212}
213
214/* Here's the write path which can sleep for
215 * page faults
216 */
217
218static inline int
219slow_user_write(struct io_mapping *mapping,
220 loff_t page_base, int page_offset,
221 char __user *user_data,
222 int length)
223{
224 char __iomem *vaddr;
225 unsigned long unwritten;
226
227 vaddr = io_mapping_map_wc(mapping, page_base);
228 if (vaddr == NULL)
229 return -EFAULT;
230 unwritten = __copy_from_user(vaddr + page_offset,
231 user_data, length);
232 io_mapping_unmap(vaddr);
233 if (unwritten)
234 return -EFAULT;
235 return 0;
Linus Torvalds9b7530cc2008-10-20 14:16:43 -0700236}
237
Eric Anholt673a3942008-07-30 12:06:12 -0700238static int
239i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
240 struct drm_i915_gem_pwrite *args,
241 struct drm_file *file_priv)
242{
243 struct drm_i915_gem_object *obj_priv = obj->driver_private;
Keith Packard0839ccb2008-10-30 19:38:48 -0700244 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -0700245 ssize_t remain;
Keith Packard0839ccb2008-10-30 19:38:48 -0700246 loff_t offset, page_base;
Eric Anholt673a3942008-07-30 12:06:12 -0700247 char __user *user_data;
Keith Packard0839ccb2008-10-30 19:38:48 -0700248 int page_offset, page_length;
249 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -0700250
251 user_data = (char __user *) (uintptr_t) args->data_ptr;
252 remain = args->size;
253 if (!access_ok(VERIFY_READ, user_data, remain))
254 return -EFAULT;
255
256
257 mutex_lock(&dev->struct_mutex);
258 ret = i915_gem_object_pin(obj, 0);
259 if (ret) {
260 mutex_unlock(&dev->struct_mutex);
261 return ret;
262 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800263 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
Eric Anholt673a3942008-07-30 12:06:12 -0700264 if (ret)
265 goto fail;
266
267 obj_priv = obj->driver_private;
268 offset = obj_priv->gtt_offset + args->offset;
269 obj_priv->dirty = 1;
270
271 while (remain > 0) {
272 /* Operation in this page
273 *
Keith Packard0839ccb2008-10-30 19:38:48 -0700274 * page_base = page offset within aperture
275 * page_offset = offset within page
276 * page_length = bytes to copy for this page
Eric Anholt673a3942008-07-30 12:06:12 -0700277 */
Keith Packard0839ccb2008-10-30 19:38:48 -0700278 page_base = (offset & ~(PAGE_SIZE-1));
279 page_offset = offset & (PAGE_SIZE-1);
280 page_length = remain;
281 if ((page_offset + remain) > PAGE_SIZE)
282 page_length = PAGE_SIZE - page_offset;
Eric Anholt673a3942008-07-30 12:06:12 -0700283
Keith Packard0839ccb2008-10-30 19:38:48 -0700284 ret = fast_user_write (dev_priv->mm.gtt_mapping, page_base,
285 page_offset, user_data, page_length);
Eric Anholt673a3942008-07-30 12:06:12 -0700286
Keith Packard0839ccb2008-10-30 19:38:48 -0700287 /* If we get a fault while copying data, then (presumably) our
288 * source page isn't available. In this case, use the
289 * non-atomic function
290 */
291 if (ret) {
292 ret = slow_user_write (dev_priv->mm.gtt_mapping,
293 page_base, page_offset,
294 user_data, page_length);
295 if (ret)
Eric Anholt673a3942008-07-30 12:06:12 -0700296 goto fail;
Eric Anholt673a3942008-07-30 12:06:12 -0700297 }
298
Keith Packard0839ccb2008-10-30 19:38:48 -0700299 remain -= page_length;
300 user_data += page_length;
301 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -0700302 }
Eric Anholt673a3942008-07-30 12:06:12 -0700303
304fail:
305 i915_gem_object_unpin(obj);
306 mutex_unlock(&dev->struct_mutex);
307
308 return ret;
309}
310
Eric Anholt3043c602008-10-02 12:24:47 -0700311static int
Eric Anholt673a3942008-07-30 12:06:12 -0700312i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
313 struct drm_i915_gem_pwrite *args,
314 struct drm_file *file_priv)
315{
316 int ret;
317 loff_t offset;
318 ssize_t written;
319
320 mutex_lock(&dev->struct_mutex);
321
Eric Anholte47c68e2008-11-14 13:35:19 -0800322 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
Eric Anholt673a3942008-07-30 12:06:12 -0700323 if (ret) {
324 mutex_unlock(&dev->struct_mutex);
325 return ret;
326 }
327
328 offset = args->offset;
329
330 written = vfs_write(obj->filp,
331 (char __user *)(uintptr_t) args->data_ptr,
332 args->size, &offset);
333 if (written != args->size) {
334 mutex_unlock(&dev->struct_mutex);
335 if (written < 0)
336 return written;
337 else
338 return -EINVAL;
339 }
340
341 mutex_unlock(&dev->struct_mutex);
342
343 return 0;
344}
345
346/**
347 * Writes data to the object referenced by handle.
348 *
349 * On error, the contents of the buffer that were to be modified are undefined.
350 */
351int
352i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
353 struct drm_file *file_priv)
354{
355 struct drm_i915_gem_pwrite *args = data;
356 struct drm_gem_object *obj;
357 struct drm_i915_gem_object *obj_priv;
358 int ret = 0;
359
360 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
361 if (obj == NULL)
362 return -EBADF;
363 obj_priv = obj->driver_private;
364
365 /* Bounds check destination.
366 *
367 * XXX: This could use review for overflow issues...
368 */
369 if (args->offset > obj->size || args->size > obj->size ||
370 args->offset + args->size > obj->size) {
371 drm_gem_object_unreference(obj);
372 return -EINVAL;
373 }
374
375 /* We can only do the GTT pwrite on untiled buffers, as otherwise
376 * it would end up going through the fenced access, and we'll get
377 * different detiling behavior between reading and writing.
378 * pread/pwrite currently are reading and writing from the CPU
379 * perspective, requiring manual detiling by the client.
380 */
381 if (obj_priv->tiling_mode == I915_TILING_NONE &&
382 dev->gtt_total != 0)
383 ret = i915_gem_gtt_pwrite(dev, obj, args, file_priv);
384 else
385 ret = i915_gem_shmem_pwrite(dev, obj, args, file_priv);
386
387#if WATCH_PWRITE
388 if (ret)
389 DRM_INFO("pwrite failed %d\n", ret);
390#endif
391
392 drm_gem_object_unreference(obj);
393
394 return ret;
395}
396
397/**
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800398 * Called when user space prepares to use an object with the CPU, either
399 * through the mmap ioctl's mapping or a GTT mapping.
Eric Anholt673a3942008-07-30 12:06:12 -0700400 */
401int
402i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
403 struct drm_file *file_priv)
404{
405 struct drm_i915_gem_set_domain *args = data;
406 struct drm_gem_object *obj;
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800407 uint32_t read_domains = args->read_domains;
408 uint32_t write_domain = args->write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -0700409 int ret;
410
411 if (!(dev->driver->driver_features & DRIVER_GEM))
412 return -ENODEV;
413
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800414 /* Only handle setting domains to types used by the CPU. */
415 if (write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
416 return -EINVAL;
417
418 if (read_domains & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
419 return -EINVAL;
420
421 /* Having something in the write domain implies it's in the read
422 * domain, and only that read domain. Enforce that in the request.
423 */
424 if (write_domain != 0 && read_domains != write_domain)
425 return -EINVAL;
426
Eric Anholt673a3942008-07-30 12:06:12 -0700427 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
428 if (obj == NULL)
429 return -EBADF;
430
431 mutex_lock(&dev->struct_mutex);
432#if WATCH_BUF
433 DRM_INFO("set_domain_ioctl %p(%d), %08x %08x\n",
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800434 obj, obj->size, read_domains, write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -0700435#endif
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800436 if (read_domains & I915_GEM_DOMAIN_GTT) {
437 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
438 } else {
Eric Anholte47c68e2008-11-14 13:35:19 -0800439 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
Eric Anholt2ef7eea2008-11-10 10:53:25 -0800440 }
441
Eric Anholt673a3942008-07-30 12:06:12 -0700442 drm_gem_object_unreference(obj);
443 mutex_unlock(&dev->struct_mutex);
444 return ret;
445}
446
447/**
448 * Called when user space has done writes to this buffer
449 */
450int
451i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
452 struct drm_file *file_priv)
453{
454 struct drm_i915_gem_sw_finish *args = data;
455 struct drm_gem_object *obj;
456 struct drm_i915_gem_object *obj_priv;
457 int ret = 0;
458
459 if (!(dev->driver->driver_features & DRIVER_GEM))
460 return -ENODEV;
461
462 mutex_lock(&dev->struct_mutex);
463 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
464 if (obj == NULL) {
465 mutex_unlock(&dev->struct_mutex);
466 return -EBADF;
467 }
468
469#if WATCH_BUF
470 DRM_INFO("%s: sw_finish %d (%p %d)\n",
471 __func__, args->handle, obj, obj->size);
472#endif
473 obj_priv = obj->driver_private;
474
475 /* Pinned buffers may be scanout, so flush the cache */
Eric Anholte47c68e2008-11-14 13:35:19 -0800476 if (obj_priv->pin_count)
477 i915_gem_object_flush_cpu_write_domain(obj);
478
Eric Anholt673a3942008-07-30 12:06:12 -0700479 drm_gem_object_unreference(obj);
480 mutex_unlock(&dev->struct_mutex);
481 return ret;
482}
483
484/**
485 * Maps the contents of an object, returning the address it is mapped
486 * into.
487 *
488 * While the mapping holds a reference on the contents of the object, it doesn't
489 * imply a ref on the object itself.
490 */
491int
492i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
493 struct drm_file *file_priv)
494{
495 struct drm_i915_gem_mmap *args = data;
496 struct drm_gem_object *obj;
497 loff_t offset;
498 unsigned long addr;
499
500 if (!(dev->driver->driver_features & DRIVER_GEM))
501 return -ENODEV;
502
503 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
504 if (obj == NULL)
505 return -EBADF;
506
507 offset = args->offset;
508
509 down_write(&current->mm->mmap_sem);
510 addr = do_mmap(obj->filp, 0, args->size,
511 PROT_READ | PROT_WRITE, MAP_SHARED,
512 args->offset);
513 up_write(&current->mm->mmap_sem);
514 mutex_lock(&dev->struct_mutex);
515 drm_gem_object_unreference(obj);
516 mutex_unlock(&dev->struct_mutex);
517 if (IS_ERR((void *)addr))
518 return addr;
519
520 args->addr_ptr = (uint64_t) addr;
521
522 return 0;
523}
524
525static void
526i915_gem_object_free_page_list(struct drm_gem_object *obj)
527{
528 struct drm_i915_gem_object *obj_priv = obj->driver_private;
529 int page_count = obj->size / PAGE_SIZE;
530 int i;
531
532 if (obj_priv->page_list == NULL)
533 return;
534
535
536 for (i = 0; i < page_count; i++)
537 if (obj_priv->page_list[i] != NULL) {
538 if (obj_priv->dirty)
539 set_page_dirty(obj_priv->page_list[i]);
540 mark_page_accessed(obj_priv->page_list[i]);
541 page_cache_release(obj_priv->page_list[i]);
542 }
543 obj_priv->dirty = 0;
544
545 drm_free(obj_priv->page_list,
546 page_count * sizeof(struct page *),
547 DRM_MEM_DRIVER);
548 obj_priv->page_list = NULL;
549}
550
551static void
Eric Anholtce44b0e2008-11-06 16:00:31 -0800552i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
Eric Anholt673a3942008-07-30 12:06:12 -0700553{
554 struct drm_device *dev = obj->dev;
555 drm_i915_private_t *dev_priv = dev->dev_private;
556 struct drm_i915_gem_object *obj_priv = obj->driver_private;
557
558 /* Add a reference if we're newly entering the active list. */
559 if (!obj_priv->active) {
560 drm_gem_object_reference(obj);
561 obj_priv->active = 1;
562 }
563 /* Move from whatever list we were on to the tail of execution. */
564 list_move_tail(&obj_priv->list,
565 &dev_priv->mm.active_list);
Eric Anholtce44b0e2008-11-06 16:00:31 -0800566 obj_priv->last_rendering_seqno = seqno;
Eric Anholt673a3942008-07-30 12:06:12 -0700567}
568
Eric Anholtce44b0e2008-11-06 16:00:31 -0800569static void
570i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
571{
572 struct drm_device *dev = obj->dev;
573 drm_i915_private_t *dev_priv = dev->dev_private;
574 struct drm_i915_gem_object *obj_priv = obj->driver_private;
575
576 BUG_ON(!obj_priv->active);
577 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list);
578 obj_priv->last_rendering_seqno = 0;
579}
Eric Anholt673a3942008-07-30 12:06:12 -0700580
581static void
582i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
583{
584 struct drm_device *dev = obj->dev;
585 drm_i915_private_t *dev_priv = dev->dev_private;
586 struct drm_i915_gem_object *obj_priv = obj->driver_private;
587
588 i915_verify_inactive(dev, __FILE__, __LINE__);
589 if (obj_priv->pin_count != 0)
590 list_del_init(&obj_priv->list);
591 else
592 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
593
Eric Anholtce44b0e2008-11-06 16:00:31 -0800594 obj_priv->last_rendering_seqno = 0;
Eric Anholt673a3942008-07-30 12:06:12 -0700595 if (obj_priv->active) {
596 obj_priv->active = 0;
597 drm_gem_object_unreference(obj);
598 }
599 i915_verify_inactive(dev, __FILE__, __LINE__);
600}
601
602/**
603 * Creates a new sequence number, emitting a write of it to the status page
604 * plus an interrupt, which will trigger i915_user_interrupt_handler.
605 *
606 * Must be called with struct_lock held.
607 *
608 * Returned sequence numbers are nonzero on success.
609 */
610static uint32_t
611i915_add_request(struct drm_device *dev, uint32_t flush_domains)
612{
613 drm_i915_private_t *dev_priv = dev->dev_private;
614 struct drm_i915_gem_request *request;
615 uint32_t seqno;
616 int was_empty;
617 RING_LOCALS;
618
619 request = drm_calloc(1, sizeof(*request), DRM_MEM_DRIVER);
620 if (request == NULL)
621 return 0;
622
623 /* Grab the seqno we're going to make this request be, and bump the
624 * next (skipping 0 so it can be the reserved no-seqno value).
625 */
626 seqno = dev_priv->mm.next_gem_seqno;
627 dev_priv->mm.next_gem_seqno++;
628 if (dev_priv->mm.next_gem_seqno == 0)
629 dev_priv->mm.next_gem_seqno++;
630
631 BEGIN_LP_RING(4);
632 OUT_RING(MI_STORE_DWORD_INDEX);
633 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
634 OUT_RING(seqno);
635
636 OUT_RING(MI_USER_INTERRUPT);
637 ADVANCE_LP_RING();
638
639 DRM_DEBUG("%d\n", seqno);
640
641 request->seqno = seqno;
642 request->emitted_jiffies = jiffies;
Eric Anholt673a3942008-07-30 12:06:12 -0700643 was_empty = list_empty(&dev_priv->mm.request_list);
644 list_add_tail(&request->list, &dev_priv->mm.request_list);
645
Eric Anholtce44b0e2008-11-06 16:00:31 -0800646 /* Associate any objects on the flushing list matching the write
647 * domain we're flushing with our flush.
648 */
649 if (flush_domains != 0) {
650 struct drm_i915_gem_object *obj_priv, *next;
651
652 list_for_each_entry_safe(obj_priv, next,
653 &dev_priv->mm.flushing_list, list) {
654 struct drm_gem_object *obj = obj_priv->obj;
655
656 if ((obj->write_domain & flush_domains) ==
657 obj->write_domain) {
658 obj->write_domain = 0;
659 i915_gem_object_move_to_active(obj, seqno);
660 }
661 }
662
663 }
664
Keith Packard6dbe2772008-10-14 21:41:13 -0700665 if (was_empty && !dev_priv->mm.suspended)
Eric Anholt673a3942008-07-30 12:06:12 -0700666 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
667 return seqno;
668}
669
670/**
671 * Command execution barrier
672 *
673 * Ensures that all commands in the ring are finished
674 * before signalling the CPU
675 */
Eric Anholt3043c602008-10-02 12:24:47 -0700676static uint32_t
Eric Anholt673a3942008-07-30 12:06:12 -0700677i915_retire_commands(struct drm_device *dev)
678{
679 drm_i915_private_t *dev_priv = dev->dev_private;
680 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
681 uint32_t flush_domains = 0;
682 RING_LOCALS;
683
684 /* The sampler always gets flushed on i965 (sigh) */
685 if (IS_I965G(dev))
686 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
687 BEGIN_LP_RING(2);
688 OUT_RING(cmd);
689 OUT_RING(0); /* noop */
690 ADVANCE_LP_RING();
691 return flush_domains;
692}
693
694/**
695 * Moves buffers associated only with the given active seqno from the active
696 * to inactive list, potentially freeing them.
697 */
698static void
699i915_gem_retire_request(struct drm_device *dev,
700 struct drm_i915_gem_request *request)
701{
702 drm_i915_private_t *dev_priv = dev->dev_private;
703
704 /* Move any buffers on the active list that are no longer referenced
705 * by the ringbuffer to the flushing/inactive lists as appropriate.
706 */
707 while (!list_empty(&dev_priv->mm.active_list)) {
708 struct drm_gem_object *obj;
709 struct drm_i915_gem_object *obj_priv;
710
711 obj_priv = list_first_entry(&dev_priv->mm.active_list,
712 struct drm_i915_gem_object,
713 list);
714 obj = obj_priv->obj;
715
716 /* If the seqno being retired doesn't match the oldest in the
717 * list, then the oldest in the list must still be newer than
718 * this seqno.
719 */
720 if (obj_priv->last_rendering_seqno != request->seqno)
721 return;
722#if WATCH_LRU
723 DRM_INFO("%s: retire %d moves to inactive list %p\n",
724 __func__, request->seqno, obj);
725#endif
726
Eric Anholtce44b0e2008-11-06 16:00:31 -0800727 if (obj->write_domain != 0)
728 i915_gem_object_move_to_flushing(obj);
729 else
Eric Anholt673a3942008-07-30 12:06:12 -0700730 i915_gem_object_move_to_inactive(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700731 }
732}
733
734/**
735 * Returns true if seq1 is later than seq2.
736 */
737static int
738i915_seqno_passed(uint32_t seq1, uint32_t seq2)
739{
740 return (int32_t)(seq1 - seq2) >= 0;
741}
742
743uint32_t
744i915_get_gem_seqno(struct drm_device *dev)
745{
746 drm_i915_private_t *dev_priv = dev->dev_private;
747
748 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
749}
750
751/**
752 * This function clears the request list as sequence numbers are passed.
753 */
754void
755i915_gem_retire_requests(struct drm_device *dev)
756{
757 drm_i915_private_t *dev_priv = dev->dev_private;
758 uint32_t seqno;
759
760 seqno = i915_get_gem_seqno(dev);
761
762 while (!list_empty(&dev_priv->mm.request_list)) {
763 struct drm_i915_gem_request *request;
764 uint32_t retiring_seqno;
765
766 request = list_first_entry(&dev_priv->mm.request_list,
767 struct drm_i915_gem_request,
768 list);
769 retiring_seqno = request->seqno;
770
771 if (i915_seqno_passed(seqno, retiring_seqno) ||
772 dev_priv->mm.wedged) {
773 i915_gem_retire_request(dev, request);
774
775 list_del(&request->list);
776 drm_free(request, sizeof(*request), DRM_MEM_DRIVER);
777 } else
778 break;
779 }
780}
781
782void
783i915_gem_retire_work_handler(struct work_struct *work)
784{
785 drm_i915_private_t *dev_priv;
786 struct drm_device *dev;
787
788 dev_priv = container_of(work, drm_i915_private_t,
789 mm.retire_work.work);
790 dev = dev_priv->dev;
791
792 mutex_lock(&dev->struct_mutex);
793 i915_gem_retire_requests(dev);
Keith Packard6dbe2772008-10-14 21:41:13 -0700794 if (!dev_priv->mm.suspended &&
795 !list_empty(&dev_priv->mm.request_list))
Eric Anholt673a3942008-07-30 12:06:12 -0700796 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
797 mutex_unlock(&dev->struct_mutex);
798}
799
800/**
801 * Waits for a sequence number to be signaled, and cleans up the
802 * request and object lists appropriately for that event.
803 */
Eric Anholt3043c602008-10-02 12:24:47 -0700804static int
Eric Anholt673a3942008-07-30 12:06:12 -0700805i915_wait_request(struct drm_device *dev, uint32_t seqno)
806{
807 drm_i915_private_t *dev_priv = dev->dev_private;
808 int ret = 0;
809
810 BUG_ON(seqno == 0);
811
812 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
813 dev_priv->mm.waiting_gem_seqno = seqno;
814 i915_user_irq_get(dev);
815 ret = wait_event_interruptible(dev_priv->irq_queue,
816 i915_seqno_passed(i915_get_gem_seqno(dev),
817 seqno) ||
818 dev_priv->mm.wedged);
819 i915_user_irq_put(dev);
820 dev_priv->mm.waiting_gem_seqno = 0;
821 }
822 if (dev_priv->mm.wedged)
823 ret = -EIO;
824
825 if (ret && ret != -ERESTARTSYS)
826 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
827 __func__, ret, seqno, i915_get_gem_seqno(dev));
828
829 /* Directly dispatch request retiring. While we have the work queue
830 * to handle this, the waiter on a request often wants an associated
831 * buffer to have made it to the inactive list, and we would need
832 * a separate wait queue to handle that.
833 */
834 if (ret == 0)
835 i915_gem_retire_requests(dev);
836
837 return ret;
838}
839
840static void
841i915_gem_flush(struct drm_device *dev,
842 uint32_t invalidate_domains,
843 uint32_t flush_domains)
844{
845 drm_i915_private_t *dev_priv = dev->dev_private;
846 uint32_t cmd;
847 RING_LOCALS;
848
849#if WATCH_EXEC
850 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
851 invalidate_domains, flush_domains);
852#endif
853
854 if (flush_domains & I915_GEM_DOMAIN_CPU)
855 drm_agp_chipset_flush(dev);
856
857 if ((invalidate_domains | flush_domains) & ~(I915_GEM_DOMAIN_CPU |
858 I915_GEM_DOMAIN_GTT)) {
859 /*
860 * read/write caches:
861 *
862 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
863 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
864 * also flushed at 2d versus 3d pipeline switches.
865 *
866 * read-only caches:
867 *
868 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
869 * MI_READ_FLUSH is set, and is always flushed on 965.
870 *
871 * I915_GEM_DOMAIN_COMMAND may not exist?
872 *
873 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
874 * invalidated when MI_EXE_FLUSH is set.
875 *
876 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
877 * invalidated with every MI_FLUSH.
878 *
879 * TLBs:
880 *
881 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
882 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
883 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
884 * are flushed at any MI_FLUSH.
885 */
886
887 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
888 if ((invalidate_domains|flush_domains) &
889 I915_GEM_DOMAIN_RENDER)
890 cmd &= ~MI_NO_WRITE_FLUSH;
891 if (!IS_I965G(dev)) {
892 /*
893 * On the 965, the sampler cache always gets flushed
894 * and this bit is reserved.
895 */
896 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
897 cmd |= MI_READ_FLUSH;
898 }
899 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
900 cmd |= MI_EXE_FLUSH;
901
902#if WATCH_EXEC
903 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
904#endif
905 BEGIN_LP_RING(2);
906 OUT_RING(cmd);
907 OUT_RING(0); /* noop */
908 ADVANCE_LP_RING();
909 }
910}
911
912/**
913 * Ensures that all rendering to the object has completed and the object is
914 * safe to unbind from the GTT or access from the CPU.
915 */
916static int
917i915_gem_object_wait_rendering(struct drm_gem_object *obj)
918{
919 struct drm_device *dev = obj->dev;
920 struct drm_i915_gem_object *obj_priv = obj->driver_private;
921 int ret;
922
Eric Anholte47c68e2008-11-14 13:35:19 -0800923 /* This function only exists to support waiting for existing rendering,
924 * not for emitting required flushes.
Eric Anholt673a3942008-07-30 12:06:12 -0700925 */
Eric Anholte47c68e2008-11-14 13:35:19 -0800926 BUG_ON((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0);
Eric Anholt673a3942008-07-30 12:06:12 -0700927
928 /* If there is rendering queued on the buffer being evicted, wait for
929 * it.
930 */
931 if (obj_priv->active) {
932#if WATCH_BUF
933 DRM_INFO("%s: object %p wait for seqno %08x\n",
934 __func__, obj, obj_priv->last_rendering_seqno);
935#endif
936 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
937 if (ret != 0)
938 return ret;
939 }
940
941 return 0;
942}
943
944/**
945 * Unbinds an object from the GTT aperture.
946 */
947static int
948i915_gem_object_unbind(struct drm_gem_object *obj)
949{
950 struct drm_device *dev = obj->dev;
951 struct drm_i915_gem_object *obj_priv = obj->driver_private;
952 int ret = 0;
953
954#if WATCH_BUF
955 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
956 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
957#endif
958 if (obj_priv->gtt_space == NULL)
959 return 0;
960
961 if (obj_priv->pin_count != 0) {
962 DRM_ERROR("Attempting to unbind pinned buffer\n");
963 return -EINVAL;
964 }
965
Eric Anholt673a3942008-07-30 12:06:12 -0700966 /* Move the object to the CPU domain to ensure that
967 * any possible CPU writes while it's not in the GTT
968 * are flushed when we go to remap it. This will
969 * also ensure that all pending GPU writes are finished
970 * before we unbind.
971 */
Eric Anholte47c68e2008-11-14 13:35:19 -0800972 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
Eric Anholt673a3942008-07-30 12:06:12 -0700973 if (ret) {
Eric Anholte47c68e2008-11-14 13:35:19 -0800974 if (ret != -ERESTARTSYS)
975 DRM_ERROR("set_domain failed: %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -0700976 return ret;
977 }
978
979 if (obj_priv->agp_mem != NULL) {
980 drm_unbind_agp(obj_priv->agp_mem);
981 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
982 obj_priv->agp_mem = NULL;
983 }
984
985 BUG_ON(obj_priv->active);
986
987 i915_gem_object_free_page_list(obj);
988
989 if (obj_priv->gtt_space) {
990 atomic_dec(&dev->gtt_count);
991 atomic_sub(obj->size, &dev->gtt_memory);
992
993 drm_mm_put_block(obj_priv->gtt_space);
994 obj_priv->gtt_space = NULL;
995 }
996
997 /* Remove ourselves from the LRU list if present. */
998 if (!list_empty(&obj_priv->list))
999 list_del_init(&obj_priv->list);
1000
1001 return 0;
1002}
1003
1004static int
1005i915_gem_evict_something(struct drm_device *dev)
1006{
1007 drm_i915_private_t *dev_priv = dev->dev_private;
1008 struct drm_gem_object *obj;
1009 struct drm_i915_gem_object *obj_priv;
1010 int ret = 0;
1011
1012 for (;;) {
1013 /* If there's an inactive buffer available now, grab it
1014 * and be done.
1015 */
1016 if (!list_empty(&dev_priv->mm.inactive_list)) {
1017 obj_priv = list_first_entry(&dev_priv->mm.inactive_list,
1018 struct drm_i915_gem_object,
1019 list);
1020 obj = obj_priv->obj;
1021 BUG_ON(obj_priv->pin_count != 0);
1022#if WATCH_LRU
1023 DRM_INFO("%s: evicting %p\n", __func__, obj);
1024#endif
1025 BUG_ON(obj_priv->active);
1026
1027 /* Wait on the rendering and unbind the buffer. */
1028 ret = i915_gem_object_unbind(obj);
1029 break;
1030 }
1031
1032 /* If we didn't get anything, but the ring is still processing
1033 * things, wait for one of those things to finish and hopefully
1034 * leave us a buffer to evict.
1035 */
1036 if (!list_empty(&dev_priv->mm.request_list)) {
1037 struct drm_i915_gem_request *request;
1038
1039 request = list_first_entry(&dev_priv->mm.request_list,
1040 struct drm_i915_gem_request,
1041 list);
1042
1043 ret = i915_wait_request(dev, request->seqno);
1044 if (ret)
1045 break;
1046
1047 /* if waiting caused an object to become inactive,
1048 * then loop around and wait for it. Otherwise, we
1049 * assume that waiting freed and unbound something,
1050 * so there should now be some space in the GTT
1051 */
1052 if (!list_empty(&dev_priv->mm.inactive_list))
1053 continue;
1054 break;
1055 }
1056
1057 /* If we didn't have anything on the request list but there
1058 * are buffers awaiting a flush, emit one and try again.
1059 * When we wait on it, those buffers waiting for that flush
1060 * will get moved to inactive.
1061 */
1062 if (!list_empty(&dev_priv->mm.flushing_list)) {
1063 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1064 struct drm_i915_gem_object,
1065 list);
1066 obj = obj_priv->obj;
1067
1068 i915_gem_flush(dev,
1069 obj->write_domain,
1070 obj->write_domain);
1071 i915_add_request(dev, obj->write_domain);
1072
1073 obj = NULL;
1074 continue;
1075 }
1076
1077 DRM_ERROR("inactive empty %d request empty %d "
1078 "flushing empty %d\n",
1079 list_empty(&dev_priv->mm.inactive_list),
1080 list_empty(&dev_priv->mm.request_list),
1081 list_empty(&dev_priv->mm.flushing_list));
1082 /* If we didn't do any of the above, there's nothing to be done
1083 * and we just can't fit it in.
1084 */
1085 return -ENOMEM;
1086 }
1087 return ret;
1088}
1089
1090static int
Keith Packardac94a962008-11-20 23:30:27 -08001091i915_gem_evict_everything(struct drm_device *dev)
1092{
1093 int ret;
1094
1095 for (;;) {
1096 ret = i915_gem_evict_something(dev);
1097 if (ret != 0)
1098 break;
1099 }
1100 return ret;
1101}
1102
1103static int
Eric Anholt673a3942008-07-30 12:06:12 -07001104i915_gem_object_get_page_list(struct drm_gem_object *obj)
1105{
1106 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1107 int page_count, i;
1108 struct address_space *mapping;
1109 struct inode *inode;
1110 struct page *page;
1111 int ret;
1112
1113 if (obj_priv->page_list)
1114 return 0;
1115
1116 /* Get the list of pages out of our struct file. They'll be pinned
1117 * at this point until we release them.
1118 */
1119 page_count = obj->size / PAGE_SIZE;
1120 BUG_ON(obj_priv->page_list != NULL);
1121 obj_priv->page_list = drm_calloc(page_count, sizeof(struct page *),
1122 DRM_MEM_DRIVER);
1123 if (obj_priv->page_list == NULL) {
1124 DRM_ERROR("Faled to allocate page list\n");
1125 return -ENOMEM;
1126 }
1127
1128 inode = obj->filp->f_path.dentry->d_inode;
1129 mapping = inode->i_mapping;
1130 for (i = 0; i < page_count; i++) {
1131 page = read_mapping_page(mapping, i, NULL);
1132 if (IS_ERR(page)) {
1133 ret = PTR_ERR(page);
1134 DRM_ERROR("read_mapping_page failed: %d\n", ret);
1135 i915_gem_object_free_page_list(obj);
1136 return ret;
1137 }
1138 obj_priv->page_list[i] = page;
1139 }
1140 return 0;
1141}
1142
1143/**
1144 * Finds free space in the GTT aperture and binds the object there.
1145 */
1146static int
1147i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
1148{
1149 struct drm_device *dev = obj->dev;
1150 drm_i915_private_t *dev_priv = dev->dev_private;
1151 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1152 struct drm_mm_node *free_space;
1153 int page_count, ret;
1154
1155 if (alignment == 0)
1156 alignment = PAGE_SIZE;
1157 if (alignment & (PAGE_SIZE - 1)) {
1158 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1159 return -EINVAL;
1160 }
1161
1162 search_free:
1163 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1164 obj->size, alignment, 0);
1165 if (free_space != NULL) {
1166 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
1167 alignment);
1168 if (obj_priv->gtt_space != NULL) {
1169 obj_priv->gtt_space->private = obj;
1170 obj_priv->gtt_offset = obj_priv->gtt_space->start;
1171 }
1172 }
1173 if (obj_priv->gtt_space == NULL) {
1174 /* If the gtt is empty and we're still having trouble
1175 * fitting our object in, we're out of memory.
1176 */
1177#if WATCH_LRU
1178 DRM_INFO("%s: GTT full, evicting something\n", __func__);
1179#endif
1180 if (list_empty(&dev_priv->mm.inactive_list) &&
1181 list_empty(&dev_priv->mm.flushing_list) &&
1182 list_empty(&dev_priv->mm.active_list)) {
1183 DRM_ERROR("GTT full, but LRU list empty\n");
1184 return -ENOMEM;
1185 }
1186
1187 ret = i915_gem_evict_something(dev);
1188 if (ret != 0) {
Keith Packardac94a962008-11-20 23:30:27 -08001189 if (ret != -ERESTARTSYS)
1190 DRM_ERROR("Failed to evict a buffer %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07001191 return ret;
1192 }
1193 goto search_free;
1194 }
1195
1196#if WATCH_BUF
1197 DRM_INFO("Binding object of size %d at 0x%08x\n",
1198 obj->size, obj_priv->gtt_offset);
1199#endif
1200 ret = i915_gem_object_get_page_list(obj);
1201 if (ret) {
1202 drm_mm_put_block(obj_priv->gtt_space);
1203 obj_priv->gtt_space = NULL;
1204 return ret;
1205 }
1206
1207 page_count = obj->size / PAGE_SIZE;
1208 /* Create an AGP memory structure pointing at our pages, and bind it
1209 * into the GTT.
1210 */
1211 obj_priv->agp_mem = drm_agp_bind_pages(dev,
1212 obj_priv->page_list,
1213 page_count,
Keith Packardba1eb1d2008-10-14 19:55:10 -07001214 obj_priv->gtt_offset,
1215 obj_priv->agp_type);
Eric Anholt673a3942008-07-30 12:06:12 -07001216 if (obj_priv->agp_mem == NULL) {
1217 i915_gem_object_free_page_list(obj);
1218 drm_mm_put_block(obj_priv->gtt_space);
1219 obj_priv->gtt_space = NULL;
1220 return -ENOMEM;
1221 }
1222 atomic_inc(&dev->gtt_count);
1223 atomic_add(obj->size, &dev->gtt_memory);
1224
1225 /* Assert that the object is not currently in any GPU domain. As it
1226 * wasn't in the GTT, there shouldn't be any way it could have been in
1227 * a GPU cache
1228 */
1229 BUG_ON(obj->read_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
1230 BUG_ON(obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
1231
1232 return 0;
1233}
1234
1235void
1236i915_gem_clflush_object(struct drm_gem_object *obj)
1237{
1238 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1239
1240 /* If we don't have a page list set up, then we're not pinned
1241 * to GPU, and we can ignore the cache flush because it'll happen
1242 * again at bind time.
1243 */
1244 if (obj_priv->page_list == NULL)
1245 return;
1246
1247 drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE);
1248}
1249
Eric Anholte47c68e2008-11-14 13:35:19 -08001250/** Flushes any GPU write domain for the object if it's dirty. */
1251static void
1252i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
1253{
1254 struct drm_device *dev = obj->dev;
1255 uint32_t seqno;
1256
1257 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
1258 return;
1259
1260 /* Queue the GPU write cache flushing we need. */
1261 i915_gem_flush(dev, 0, obj->write_domain);
1262 seqno = i915_add_request(dev, obj->write_domain);
1263 obj->write_domain = 0;
1264 i915_gem_object_move_to_active(obj, seqno);
1265}
1266
1267/** Flushes the GTT write domain for the object if it's dirty. */
1268static void
1269i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
1270{
1271 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
1272 return;
1273
1274 /* No actual flushing is required for the GTT write domain. Writes
1275 * to it immediately go to main memory as far as we know, so there's
1276 * no chipset flush. It also doesn't land in render cache.
1277 */
1278 obj->write_domain = 0;
1279}
1280
1281/** Flushes the CPU write domain for the object if it's dirty. */
1282static void
1283i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
1284{
1285 struct drm_device *dev = obj->dev;
1286
1287 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
1288 return;
1289
1290 i915_gem_clflush_object(obj);
1291 drm_agp_chipset_flush(dev);
1292 obj->write_domain = 0;
1293}
1294
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001295/**
1296 * Moves a single object to the GTT read, and possibly write domain.
1297 *
1298 * This function returns when the move is complete, including waiting on
1299 * flushes to occur.
1300 */
1301static int
1302i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
1303{
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001304 struct drm_i915_gem_object *obj_priv = obj->driver_private;
Eric Anholte47c68e2008-11-14 13:35:19 -08001305 int ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001306
Eric Anholte47c68e2008-11-14 13:35:19 -08001307 i915_gem_object_flush_gpu_write_domain(obj);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001308 /* Wait on any GPU rendering and flushing to occur. */
Eric Anholte47c68e2008-11-14 13:35:19 -08001309 ret = i915_gem_object_wait_rendering(obj);
1310 if (ret != 0)
1311 return ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001312
1313 /* If we're writing through the GTT domain, then CPU and GPU caches
1314 * will need to be invalidated at next use.
1315 */
1316 if (write)
Eric Anholte47c68e2008-11-14 13:35:19 -08001317 obj->read_domains &= I915_GEM_DOMAIN_GTT;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001318
Eric Anholte47c68e2008-11-14 13:35:19 -08001319 i915_gem_object_flush_cpu_write_domain(obj);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001320
1321 /* It should now be out of any other write domains, and we can update
1322 * the domain values for our changes.
1323 */
1324 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
1325 obj->read_domains |= I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08001326 if (write) {
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001327 obj->write_domain = I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08001328 obj_priv->dirty = 1;
1329 }
1330
1331 return 0;
1332}
1333
1334/**
1335 * Moves a single object to the CPU read, and possibly write domain.
1336 *
1337 * This function returns when the move is complete, including waiting on
1338 * flushes to occur.
1339 */
1340static int
1341i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
1342{
1343 struct drm_device *dev = obj->dev;
1344 int ret;
1345
1346 i915_gem_object_flush_gpu_write_domain(obj);
1347 /* Wait on any GPU rendering and flushing to occur. */
1348 ret = i915_gem_object_wait_rendering(obj);
1349 if (ret != 0)
1350 return ret;
1351
1352 i915_gem_object_flush_gtt_write_domain(obj);
1353
1354 /* If we have a partially-valid cache of the object in the CPU,
1355 * finish invalidating it and free the per-page flags.
1356 */
1357 i915_gem_object_set_to_full_cpu_read_domain(obj);
1358
1359 /* Flush the CPU cache if it's still invalid. */
1360 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
1361 i915_gem_clflush_object(obj);
1362 drm_agp_chipset_flush(dev);
1363
1364 obj->read_domains |= I915_GEM_DOMAIN_CPU;
1365 }
1366
1367 /* It should now be out of any other write domains, and we can update
1368 * the domain values for our changes.
1369 */
1370 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
1371
1372 /* If we're writing through the CPU, then the GPU read domains will
1373 * need to be invalidated at next use.
1374 */
1375 if (write) {
1376 obj->read_domains &= I915_GEM_DOMAIN_CPU;
1377 obj->write_domain = I915_GEM_DOMAIN_CPU;
1378 }
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001379
1380 return 0;
1381}
1382
Eric Anholt673a3942008-07-30 12:06:12 -07001383/*
1384 * Set the next domain for the specified object. This
1385 * may not actually perform the necessary flushing/invaliding though,
1386 * as that may want to be batched with other set_domain operations
1387 *
1388 * This is (we hope) the only really tricky part of gem. The goal
1389 * is fairly simple -- track which caches hold bits of the object
1390 * and make sure they remain coherent. A few concrete examples may
1391 * help to explain how it works. For shorthand, we use the notation
1392 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
1393 * a pair of read and write domain masks.
1394 *
1395 * Case 1: the batch buffer
1396 *
1397 * 1. Allocated
1398 * 2. Written by CPU
1399 * 3. Mapped to GTT
1400 * 4. Read by GPU
1401 * 5. Unmapped from GTT
1402 * 6. Freed
1403 *
1404 * Let's take these a step at a time
1405 *
1406 * 1. Allocated
1407 * Pages allocated from the kernel may still have
1408 * cache contents, so we set them to (CPU, CPU) always.
1409 * 2. Written by CPU (using pwrite)
1410 * The pwrite function calls set_domain (CPU, CPU) and
1411 * this function does nothing (as nothing changes)
1412 * 3. Mapped by GTT
1413 * This function asserts that the object is not
1414 * currently in any GPU-based read or write domains
1415 * 4. Read by GPU
1416 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
1417 * As write_domain is zero, this function adds in the
1418 * current read domains (CPU+COMMAND, 0).
1419 * flush_domains is set to CPU.
1420 * invalidate_domains is set to COMMAND
1421 * clflush is run to get data out of the CPU caches
1422 * then i915_dev_set_domain calls i915_gem_flush to
1423 * emit an MI_FLUSH and drm_agp_chipset_flush
1424 * 5. Unmapped from GTT
1425 * i915_gem_object_unbind calls set_domain (CPU, CPU)
1426 * flush_domains and invalidate_domains end up both zero
1427 * so no flushing/invalidating happens
1428 * 6. Freed
1429 * yay, done
1430 *
1431 * Case 2: The shared render buffer
1432 *
1433 * 1. Allocated
1434 * 2. Mapped to GTT
1435 * 3. Read/written by GPU
1436 * 4. set_domain to (CPU,CPU)
1437 * 5. Read/written by CPU
1438 * 6. Read/written by GPU
1439 *
1440 * 1. Allocated
1441 * Same as last example, (CPU, CPU)
1442 * 2. Mapped to GTT
1443 * Nothing changes (assertions find that it is not in the GPU)
1444 * 3. Read/written by GPU
1445 * execbuffer calls set_domain (RENDER, RENDER)
1446 * flush_domains gets CPU
1447 * invalidate_domains gets GPU
1448 * clflush (obj)
1449 * MI_FLUSH and drm_agp_chipset_flush
1450 * 4. set_domain (CPU, CPU)
1451 * flush_domains gets GPU
1452 * invalidate_domains gets CPU
1453 * wait_rendering (obj) to make sure all drawing is complete.
1454 * This will include an MI_FLUSH to get the data from GPU
1455 * to memory
1456 * clflush (obj) to invalidate the CPU cache
1457 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
1458 * 5. Read/written by CPU
1459 * cache lines are loaded and dirtied
1460 * 6. Read written by GPU
1461 * Same as last GPU access
1462 *
1463 * Case 3: The constant buffer
1464 *
1465 * 1. Allocated
1466 * 2. Written by CPU
1467 * 3. Read by GPU
1468 * 4. Updated (written) by CPU again
1469 * 5. Read by GPU
1470 *
1471 * 1. Allocated
1472 * (CPU, CPU)
1473 * 2. Written by CPU
1474 * (CPU, CPU)
1475 * 3. Read by GPU
1476 * (CPU+RENDER, 0)
1477 * flush_domains = CPU
1478 * invalidate_domains = RENDER
1479 * clflush (obj)
1480 * MI_FLUSH
1481 * drm_agp_chipset_flush
1482 * 4. Updated (written) by CPU again
1483 * (CPU, CPU)
1484 * flush_domains = 0 (no previous write domain)
1485 * invalidate_domains = 0 (no new read domains)
1486 * 5. Read by GPU
1487 * (CPU+RENDER, 0)
1488 * flush_domains = CPU
1489 * invalidate_domains = RENDER
1490 * clflush (obj)
1491 * MI_FLUSH
1492 * drm_agp_chipset_flush
1493 */
Keith Packardc0d90822008-11-20 23:11:08 -08001494static void
1495i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
1496 uint32_t read_domains,
1497 uint32_t write_domain)
Eric Anholt673a3942008-07-30 12:06:12 -07001498{
1499 struct drm_device *dev = obj->dev;
1500 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1501 uint32_t invalidate_domains = 0;
1502 uint32_t flush_domains = 0;
Eric Anholte47c68e2008-11-14 13:35:19 -08001503
1504 BUG_ON(read_domains & I915_GEM_DOMAIN_CPU);
1505 BUG_ON(write_domain == I915_GEM_DOMAIN_CPU);
Eric Anholt673a3942008-07-30 12:06:12 -07001506
1507#if WATCH_BUF
1508 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
1509 __func__, obj,
1510 obj->read_domains, read_domains,
1511 obj->write_domain, write_domain);
1512#endif
1513 /*
1514 * If the object isn't moving to a new write domain,
1515 * let the object stay in multiple read domains
1516 */
1517 if (write_domain == 0)
1518 read_domains |= obj->read_domains;
1519 else
1520 obj_priv->dirty = 1;
1521
1522 /*
1523 * Flush the current write domain if
1524 * the new read domains don't match. Invalidate
1525 * any read domains which differ from the old
1526 * write domain
1527 */
1528 if (obj->write_domain && obj->write_domain != read_domains) {
1529 flush_domains |= obj->write_domain;
1530 invalidate_domains |= read_domains & ~obj->write_domain;
1531 }
1532 /*
1533 * Invalidate any read caches which may have
1534 * stale data. That is, any new read domains.
1535 */
1536 invalidate_domains |= read_domains & ~obj->read_domains;
1537 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
1538#if WATCH_BUF
1539 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
1540 __func__, flush_domains, invalidate_domains);
1541#endif
Eric Anholt673a3942008-07-30 12:06:12 -07001542 i915_gem_clflush_object(obj);
1543 }
1544
1545 if ((write_domain | flush_domains) != 0)
1546 obj->write_domain = write_domain;
Eric Anholt673a3942008-07-30 12:06:12 -07001547 obj->read_domains = read_domains;
1548
1549 dev->invalidate_domains |= invalidate_domains;
1550 dev->flush_domains |= flush_domains;
1551#if WATCH_BUF
1552 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
1553 __func__,
1554 obj->read_domains, obj->write_domain,
1555 dev->invalidate_domains, dev->flush_domains);
1556#endif
Eric Anholt673a3942008-07-30 12:06:12 -07001557}
1558
1559/**
Eric Anholte47c68e2008-11-14 13:35:19 -08001560 * Moves the object from a partially CPU read to a full one.
Eric Anholt673a3942008-07-30 12:06:12 -07001561 *
Eric Anholte47c68e2008-11-14 13:35:19 -08001562 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
1563 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
1564 */
1565static void
1566i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
1567{
1568 struct drm_device *dev = obj->dev;
1569 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1570
1571 if (!obj_priv->page_cpu_valid)
1572 return;
1573
1574 /* If we're partially in the CPU read domain, finish moving it in.
1575 */
1576 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
1577 int i;
1578
1579 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
1580 if (obj_priv->page_cpu_valid[i])
1581 continue;
1582 drm_clflush_pages(obj_priv->page_list + i, 1);
1583 }
1584 drm_agp_chipset_flush(dev);
1585 }
1586
1587 /* Free the page_cpu_valid mappings which are now stale, whether
1588 * or not we've got I915_GEM_DOMAIN_CPU.
1589 */
1590 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
1591 DRM_MEM_DRIVER);
1592 obj_priv->page_cpu_valid = NULL;
1593}
1594
1595/**
1596 * Set the CPU read domain on a range of the object.
1597 *
1598 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
1599 * not entirely valid. The page_cpu_valid member of the object flags which
1600 * pages have been flushed, and will be respected by
1601 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
1602 * of the whole object.
1603 *
1604 * This function returns when the move is complete, including waiting on
1605 * flushes to occur.
Eric Anholt673a3942008-07-30 12:06:12 -07001606 */
1607static int
Eric Anholte47c68e2008-11-14 13:35:19 -08001608i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
1609 uint64_t offset, uint64_t size)
Eric Anholt673a3942008-07-30 12:06:12 -07001610{
1611 struct drm_i915_gem_object *obj_priv = obj->driver_private;
Eric Anholte47c68e2008-11-14 13:35:19 -08001612 int i, ret;
Eric Anholt673a3942008-07-30 12:06:12 -07001613
Eric Anholte47c68e2008-11-14 13:35:19 -08001614 if (offset == 0 && size == obj->size)
1615 return i915_gem_object_set_to_cpu_domain(obj, 0);
1616
1617 i915_gem_object_flush_gpu_write_domain(obj);
1618 /* Wait on any GPU rendering and flushing to occur. */
1619 ret = i915_gem_object_wait_rendering(obj);
1620 if (ret != 0)
1621 return ret;
1622 i915_gem_object_flush_gtt_write_domain(obj);
1623
1624 /* If we're already fully in the CPU read domain, we're done. */
1625 if (obj_priv->page_cpu_valid == NULL &&
1626 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
Eric Anholt673a3942008-07-30 12:06:12 -07001627 return 0;
1628
Eric Anholte47c68e2008-11-14 13:35:19 -08001629 /* Otherwise, create/clear the per-page CPU read domain flag if we're
1630 * newly adding I915_GEM_DOMAIN_CPU
1631 */
Eric Anholt673a3942008-07-30 12:06:12 -07001632 if (obj_priv->page_cpu_valid == NULL) {
1633 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE,
1634 DRM_MEM_DRIVER);
Eric Anholte47c68e2008-11-14 13:35:19 -08001635 if (obj_priv->page_cpu_valid == NULL)
1636 return -ENOMEM;
1637 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
1638 memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
Eric Anholt673a3942008-07-30 12:06:12 -07001639
1640 /* Flush the cache on any pages that are still invalid from the CPU's
1641 * perspective.
1642 */
Eric Anholte47c68e2008-11-14 13:35:19 -08001643 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
1644 i++) {
Eric Anholt673a3942008-07-30 12:06:12 -07001645 if (obj_priv->page_cpu_valid[i])
1646 continue;
1647
1648 drm_clflush_pages(obj_priv->page_list + i, 1);
1649
1650 obj_priv->page_cpu_valid[i] = 1;
1651 }
1652
Eric Anholte47c68e2008-11-14 13:35:19 -08001653 /* It should now be out of any other write domains, and we can update
1654 * the domain values for our changes.
1655 */
1656 BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
1657
1658 obj->read_domains |= I915_GEM_DOMAIN_CPU;
1659
Eric Anholt673a3942008-07-30 12:06:12 -07001660 return 0;
1661}
1662
1663/**
Eric Anholt673a3942008-07-30 12:06:12 -07001664 * Pin an object to the GTT and evaluate the relocations landing in it.
1665 */
1666static int
1667i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
1668 struct drm_file *file_priv,
1669 struct drm_i915_gem_exec_object *entry)
1670{
1671 struct drm_device *dev = obj->dev;
Keith Packard0839ccb2008-10-30 19:38:48 -07001672 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07001673 struct drm_i915_gem_relocation_entry reloc;
1674 struct drm_i915_gem_relocation_entry __user *relocs;
1675 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1676 int i, ret;
Keith Packard0839ccb2008-10-30 19:38:48 -07001677 void __iomem *reloc_page;
Eric Anholt673a3942008-07-30 12:06:12 -07001678
1679 /* Choose the GTT offset for our buffer and put it there. */
1680 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
1681 if (ret)
1682 return ret;
1683
1684 entry->offset = obj_priv->gtt_offset;
1685
1686 relocs = (struct drm_i915_gem_relocation_entry __user *)
1687 (uintptr_t) entry->relocs_ptr;
1688 /* Apply the relocations, using the GTT aperture to avoid cache
1689 * flushing requirements.
1690 */
1691 for (i = 0; i < entry->relocation_count; i++) {
1692 struct drm_gem_object *target_obj;
1693 struct drm_i915_gem_object *target_obj_priv;
Eric Anholt3043c602008-10-02 12:24:47 -07001694 uint32_t reloc_val, reloc_offset;
1695 uint32_t __iomem *reloc_entry;
Eric Anholt673a3942008-07-30 12:06:12 -07001696
1697 ret = copy_from_user(&reloc, relocs + i, sizeof(reloc));
1698 if (ret != 0) {
1699 i915_gem_object_unpin(obj);
1700 return ret;
1701 }
1702
1703 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
1704 reloc.target_handle);
1705 if (target_obj == NULL) {
1706 i915_gem_object_unpin(obj);
1707 return -EBADF;
1708 }
1709 target_obj_priv = target_obj->driver_private;
1710
1711 /* The target buffer should have appeared before us in the
1712 * exec_object list, so it should have a GTT space bound by now.
1713 */
1714 if (target_obj_priv->gtt_space == NULL) {
1715 DRM_ERROR("No GTT space found for object %d\n",
1716 reloc.target_handle);
1717 drm_gem_object_unreference(target_obj);
1718 i915_gem_object_unpin(obj);
1719 return -EINVAL;
1720 }
1721
1722 if (reloc.offset > obj->size - 4) {
1723 DRM_ERROR("Relocation beyond object bounds: "
1724 "obj %p target %d offset %d size %d.\n",
1725 obj, reloc.target_handle,
1726 (int) reloc.offset, (int) obj->size);
1727 drm_gem_object_unreference(target_obj);
1728 i915_gem_object_unpin(obj);
1729 return -EINVAL;
1730 }
1731 if (reloc.offset & 3) {
1732 DRM_ERROR("Relocation not 4-byte aligned: "
1733 "obj %p target %d offset %d.\n",
1734 obj, reloc.target_handle,
1735 (int) reloc.offset);
1736 drm_gem_object_unreference(target_obj);
1737 i915_gem_object_unpin(obj);
1738 return -EINVAL;
1739 }
1740
Eric Anholte47c68e2008-11-14 13:35:19 -08001741 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
1742 reloc.read_domains & I915_GEM_DOMAIN_CPU) {
1743 DRM_ERROR("reloc with read/write CPU domains: "
1744 "obj %p target %d offset %d "
1745 "read %08x write %08x",
1746 obj, reloc.target_handle,
1747 (int) reloc.offset,
1748 reloc.read_domains,
1749 reloc.write_domain);
1750 return -EINVAL;
1751 }
1752
Eric Anholt673a3942008-07-30 12:06:12 -07001753 if (reloc.write_domain && target_obj->pending_write_domain &&
1754 reloc.write_domain != target_obj->pending_write_domain) {
1755 DRM_ERROR("Write domain conflict: "
1756 "obj %p target %d offset %d "
1757 "new %08x old %08x\n",
1758 obj, reloc.target_handle,
1759 (int) reloc.offset,
1760 reloc.write_domain,
1761 target_obj->pending_write_domain);
1762 drm_gem_object_unreference(target_obj);
1763 i915_gem_object_unpin(obj);
1764 return -EINVAL;
1765 }
1766
1767#if WATCH_RELOC
1768 DRM_INFO("%s: obj %p offset %08x target %d "
1769 "read %08x write %08x gtt %08x "
1770 "presumed %08x delta %08x\n",
1771 __func__,
1772 obj,
1773 (int) reloc.offset,
1774 (int) reloc.target_handle,
1775 (int) reloc.read_domains,
1776 (int) reloc.write_domain,
1777 (int) target_obj_priv->gtt_offset,
1778 (int) reloc.presumed_offset,
1779 reloc.delta);
1780#endif
1781
1782 target_obj->pending_read_domains |= reloc.read_domains;
1783 target_obj->pending_write_domain |= reloc.write_domain;
1784
1785 /* If the relocation already has the right value in it, no
1786 * more work needs to be done.
1787 */
1788 if (target_obj_priv->gtt_offset == reloc.presumed_offset) {
1789 drm_gem_object_unreference(target_obj);
1790 continue;
1791 }
1792
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001793 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1794 if (ret != 0) {
1795 drm_gem_object_unreference(target_obj);
1796 i915_gem_object_unpin(obj);
1797 return -EINVAL;
Eric Anholt673a3942008-07-30 12:06:12 -07001798 }
1799
1800 /* Map the page containing the relocation we're going to
1801 * perform.
1802 */
1803 reloc_offset = obj_priv->gtt_offset + reloc.offset;
Keith Packard0839ccb2008-10-30 19:38:48 -07001804 reloc_page = io_mapping_map_atomic_wc(dev_priv->mm.gtt_mapping,
1805 (reloc_offset &
1806 ~(PAGE_SIZE - 1)));
Eric Anholt3043c602008-10-02 12:24:47 -07001807 reloc_entry = (uint32_t __iomem *)(reloc_page +
Keith Packard0839ccb2008-10-30 19:38:48 -07001808 (reloc_offset & (PAGE_SIZE - 1)));
Eric Anholt673a3942008-07-30 12:06:12 -07001809 reloc_val = target_obj_priv->gtt_offset + reloc.delta;
1810
1811#if WATCH_BUF
1812 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
1813 obj, (unsigned int) reloc.offset,
1814 readl(reloc_entry), reloc_val);
1815#endif
1816 writel(reloc_val, reloc_entry);
Keith Packard0839ccb2008-10-30 19:38:48 -07001817 io_mapping_unmap_atomic(reloc_page);
Eric Anholt673a3942008-07-30 12:06:12 -07001818
1819 /* Write the updated presumed offset for this entry back out
1820 * to the user.
1821 */
1822 reloc.presumed_offset = target_obj_priv->gtt_offset;
1823 ret = copy_to_user(relocs + i, &reloc, sizeof(reloc));
1824 if (ret != 0) {
1825 drm_gem_object_unreference(target_obj);
1826 i915_gem_object_unpin(obj);
1827 return ret;
1828 }
1829
1830 drm_gem_object_unreference(target_obj);
1831 }
1832
Eric Anholt673a3942008-07-30 12:06:12 -07001833#if WATCH_BUF
1834 if (0)
1835 i915_gem_dump_object(obj, 128, __func__, ~0);
1836#endif
1837 return 0;
1838}
1839
1840/** Dispatch a batchbuffer to the ring
1841 */
1842static int
1843i915_dispatch_gem_execbuffer(struct drm_device *dev,
1844 struct drm_i915_gem_execbuffer *exec,
1845 uint64_t exec_offset)
1846{
1847 drm_i915_private_t *dev_priv = dev->dev_private;
1848 struct drm_clip_rect __user *boxes = (struct drm_clip_rect __user *)
1849 (uintptr_t) exec->cliprects_ptr;
1850 int nbox = exec->num_cliprects;
1851 int i = 0, count;
1852 uint32_t exec_start, exec_len;
1853 RING_LOCALS;
1854
1855 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
1856 exec_len = (uint32_t) exec->batch_len;
1857
1858 if ((exec_start | exec_len) & 0x7) {
1859 DRM_ERROR("alignment\n");
1860 return -EINVAL;
1861 }
1862
1863 if (!exec_start)
1864 return -EINVAL;
1865
1866 count = nbox ? nbox : 1;
1867
1868 for (i = 0; i < count; i++) {
1869 if (i < nbox) {
1870 int ret = i915_emit_box(dev, boxes, i,
1871 exec->DR1, exec->DR4);
1872 if (ret)
1873 return ret;
1874 }
1875
1876 if (IS_I830(dev) || IS_845G(dev)) {
1877 BEGIN_LP_RING(4);
1878 OUT_RING(MI_BATCH_BUFFER);
1879 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1880 OUT_RING(exec_start + exec_len - 4);
1881 OUT_RING(0);
1882 ADVANCE_LP_RING();
1883 } else {
1884 BEGIN_LP_RING(2);
1885 if (IS_I965G(dev)) {
1886 OUT_RING(MI_BATCH_BUFFER_START |
1887 (2 << 6) |
1888 MI_BATCH_NON_SECURE_I965);
1889 OUT_RING(exec_start);
1890 } else {
1891 OUT_RING(MI_BATCH_BUFFER_START |
1892 (2 << 6));
1893 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1894 }
1895 ADVANCE_LP_RING();
1896 }
1897 }
1898
1899 /* XXX breadcrumb */
1900 return 0;
1901}
1902
1903/* Throttle our rendering by waiting until the ring has completed our requests
1904 * emitted over 20 msec ago.
1905 *
1906 * This should get us reasonable parallelism between CPU and GPU but also
1907 * relatively low latency when blocking on a particular request to finish.
1908 */
1909static int
1910i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
1911{
1912 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1913 int ret = 0;
1914 uint32_t seqno;
1915
1916 mutex_lock(&dev->struct_mutex);
1917 seqno = i915_file_priv->mm.last_gem_throttle_seqno;
1918 i915_file_priv->mm.last_gem_throttle_seqno =
1919 i915_file_priv->mm.last_gem_seqno;
1920 if (seqno)
1921 ret = i915_wait_request(dev, seqno);
1922 mutex_unlock(&dev->struct_mutex);
1923 return ret;
1924}
1925
1926int
1927i915_gem_execbuffer(struct drm_device *dev, void *data,
1928 struct drm_file *file_priv)
1929{
1930 drm_i915_private_t *dev_priv = dev->dev_private;
1931 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1932 struct drm_i915_gem_execbuffer *args = data;
1933 struct drm_i915_gem_exec_object *exec_list = NULL;
1934 struct drm_gem_object **object_list = NULL;
1935 struct drm_gem_object *batch_obj;
1936 int ret, i, pinned = 0;
1937 uint64_t exec_offset;
1938 uint32_t seqno, flush_domains;
Keith Packardac94a962008-11-20 23:30:27 -08001939 int pin_tries;
Eric Anholt673a3942008-07-30 12:06:12 -07001940
1941#if WATCH_EXEC
1942 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1943 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1944#endif
1945
Eric Anholt4f481ed2008-09-10 14:22:49 -07001946 if (args->buffer_count < 1) {
1947 DRM_ERROR("execbuf with %d buffers\n", args->buffer_count);
1948 return -EINVAL;
1949 }
Eric Anholt673a3942008-07-30 12:06:12 -07001950 /* Copy in the exec list from userland */
1951 exec_list = drm_calloc(sizeof(*exec_list), args->buffer_count,
1952 DRM_MEM_DRIVER);
1953 object_list = drm_calloc(sizeof(*object_list), args->buffer_count,
1954 DRM_MEM_DRIVER);
1955 if (exec_list == NULL || object_list == NULL) {
1956 DRM_ERROR("Failed to allocate exec or object list "
1957 "for %d buffers\n",
1958 args->buffer_count);
1959 ret = -ENOMEM;
1960 goto pre_mutex_err;
1961 }
1962 ret = copy_from_user(exec_list,
1963 (struct drm_i915_relocation_entry __user *)
1964 (uintptr_t) args->buffers_ptr,
1965 sizeof(*exec_list) * args->buffer_count);
1966 if (ret != 0) {
1967 DRM_ERROR("copy %d exec entries failed %d\n",
1968 args->buffer_count, ret);
1969 goto pre_mutex_err;
1970 }
1971
1972 mutex_lock(&dev->struct_mutex);
1973
1974 i915_verify_inactive(dev, __FILE__, __LINE__);
1975
1976 if (dev_priv->mm.wedged) {
1977 DRM_ERROR("Execbuf while wedged\n");
1978 mutex_unlock(&dev->struct_mutex);
1979 return -EIO;
1980 }
1981
1982 if (dev_priv->mm.suspended) {
1983 DRM_ERROR("Execbuf while VT-switched.\n");
1984 mutex_unlock(&dev->struct_mutex);
1985 return -EBUSY;
1986 }
1987
Keith Packardac94a962008-11-20 23:30:27 -08001988 /* Look up object handles */
Eric Anholt673a3942008-07-30 12:06:12 -07001989 for (i = 0; i < args->buffer_count; i++) {
1990 object_list[i] = drm_gem_object_lookup(dev, file_priv,
1991 exec_list[i].handle);
1992 if (object_list[i] == NULL) {
1993 DRM_ERROR("Invalid object handle %d at index %d\n",
1994 exec_list[i].handle, i);
1995 ret = -EBADF;
1996 goto err;
1997 }
Keith Packardac94a962008-11-20 23:30:27 -08001998 }
Eric Anholt673a3942008-07-30 12:06:12 -07001999
Keith Packardac94a962008-11-20 23:30:27 -08002000 /* Pin and relocate */
2001 for (pin_tries = 0; ; pin_tries++) {
2002 ret = 0;
2003 for (i = 0; i < args->buffer_count; i++) {
2004 object_list[i]->pending_read_domains = 0;
2005 object_list[i]->pending_write_domain = 0;
2006 ret = i915_gem_object_pin_and_relocate(object_list[i],
2007 file_priv,
2008 &exec_list[i]);
2009 if (ret)
2010 break;
2011 pinned = i + 1;
2012 }
2013 /* success */
2014 if (ret == 0)
2015 break;
2016
2017 /* error other than GTT full, or we've already tried again */
2018 if (ret != -ENOMEM || pin_tries >= 1) {
2019 DRM_ERROR("Failed to pin buffers %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07002020 goto err;
2021 }
Keith Packardac94a962008-11-20 23:30:27 -08002022
2023 /* unpin all of our buffers */
2024 for (i = 0; i < pinned; i++)
2025 i915_gem_object_unpin(object_list[i]);
2026
2027 /* evict everyone we can from the aperture */
2028 ret = i915_gem_evict_everything(dev);
2029 if (ret)
2030 goto err;
Eric Anholt673a3942008-07-30 12:06:12 -07002031 }
2032
2033 /* Set the pending read domains for the batch buffer to COMMAND */
2034 batch_obj = object_list[args->buffer_count-1];
2035 batch_obj->pending_read_domains = I915_GEM_DOMAIN_COMMAND;
2036 batch_obj->pending_write_domain = 0;
2037
2038 i915_verify_inactive(dev, __FILE__, __LINE__);
2039
Keith Packard646f0f62008-11-20 23:23:03 -08002040 /* Zero the global flush/invalidate flags. These
2041 * will be modified as new domains are computed
2042 * for each object
2043 */
2044 dev->invalidate_domains = 0;
2045 dev->flush_domains = 0;
2046
Eric Anholt673a3942008-07-30 12:06:12 -07002047 for (i = 0; i < args->buffer_count; i++) {
2048 struct drm_gem_object *obj = object_list[i];
Eric Anholt673a3942008-07-30 12:06:12 -07002049
Keith Packard646f0f62008-11-20 23:23:03 -08002050 /* Compute new gpu domains and update invalidate/flush */
Keith Packardc0d90822008-11-20 23:11:08 -08002051 i915_gem_object_set_to_gpu_domain(obj,
2052 obj->pending_read_domains,
2053 obj->pending_write_domain);
Eric Anholt673a3942008-07-30 12:06:12 -07002054 }
2055
2056 i915_verify_inactive(dev, __FILE__, __LINE__);
2057
Keith Packard646f0f62008-11-20 23:23:03 -08002058 if (dev->invalidate_domains | dev->flush_domains) {
2059#if WATCH_EXEC
2060 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
2061 __func__,
2062 dev->invalidate_domains,
2063 dev->flush_domains);
2064#endif
2065 i915_gem_flush(dev,
2066 dev->invalidate_domains,
2067 dev->flush_domains);
2068 if (dev->flush_domains)
2069 (void)i915_add_request(dev, dev->flush_domains);
2070 }
Eric Anholt673a3942008-07-30 12:06:12 -07002071
2072 i915_verify_inactive(dev, __FILE__, __LINE__);
2073
2074#if WATCH_COHERENCY
2075 for (i = 0; i < args->buffer_count; i++) {
2076 i915_gem_object_check_coherency(object_list[i],
2077 exec_list[i].handle);
2078 }
2079#endif
2080
2081 exec_offset = exec_list[args->buffer_count - 1].offset;
2082
2083#if WATCH_EXEC
2084 i915_gem_dump_object(object_list[args->buffer_count - 1],
2085 args->batch_len,
2086 __func__,
2087 ~0);
2088#endif
2089
Eric Anholt673a3942008-07-30 12:06:12 -07002090 /* Exec the batchbuffer */
2091 ret = i915_dispatch_gem_execbuffer(dev, args, exec_offset);
2092 if (ret) {
2093 DRM_ERROR("dispatch failed %d\n", ret);
2094 goto err;
2095 }
2096
2097 /*
2098 * Ensure that the commands in the batch buffer are
2099 * finished before the interrupt fires
2100 */
2101 flush_domains = i915_retire_commands(dev);
2102
2103 i915_verify_inactive(dev, __FILE__, __LINE__);
2104
2105 /*
2106 * Get a seqno representing the execution of the current buffer,
2107 * which we can wait on. We would like to mitigate these interrupts,
2108 * likely by only creating seqnos occasionally (so that we have
2109 * *some* interrupts representing completion of buffers that we can
2110 * wait on when trying to clear up gtt space).
2111 */
2112 seqno = i915_add_request(dev, flush_domains);
2113 BUG_ON(seqno == 0);
2114 i915_file_priv->mm.last_gem_seqno = seqno;
2115 for (i = 0; i < args->buffer_count; i++) {
2116 struct drm_gem_object *obj = object_list[i];
Eric Anholt673a3942008-07-30 12:06:12 -07002117
Eric Anholtce44b0e2008-11-06 16:00:31 -08002118 i915_gem_object_move_to_active(obj, seqno);
Eric Anholt673a3942008-07-30 12:06:12 -07002119#if WATCH_LRU
2120 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
2121#endif
2122 }
2123#if WATCH_LRU
2124 i915_dump_lru(dev, __func__);
2125#endif
2126
2127 i915_verify_inactive(dev, __FILE__, __LINE__);
2128
2129 /* Copy the new buffer offsets back to the user's exec list. */
2130 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
2131 (uintptr_t) args->buffers_ptr,
2132 exec_list,
2133 sizeof(*exec_list) * args->buffer_count);
2134 if (ret)
2135 DRM_ERROR("failed to copy %d exec entries "
2136 "back to user (%d)\n",
2137 args->buffer_count, ret);
2138err:
2139 if (object_list != NULL) {
2140 for (i = 0; i < pinned; i++)
2141 i915_gem_object_unpin(object_list[i]);
2142
2143 for (i = 0; i < args->buffer_count; i++)
2144 drm_gem_object_unreference(object_list[i]);
2145 }
2146 mutex_unlock(&dev->struct_mutex);
2147
2148pre_mutex_err:
2149 drm_free(object_list, sizeof(*object_list) * args->buffer_count,
2150 DRM_MEM_DRIVER);
2151 drm_free(exec_list, sizeof(*exec_list) * args->buffer_count,
2152 DRM_MEM_DRIVER);
2153
2154 return ret;
2155}
2156
2157int
2158i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
2159{
2160 struct drm_device *dev = obj->dev;
2161 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2162 int ret;
2163
2164 i915_verify_inactive(dev, __FILE__, __LINE__);
2165 if (obj_priv->gtt_space == NULL) {
2166 ret = i915_gem_object_bind_to_gtt(obj, alignment);
2167 if (ret != 0) {
2168 DRM_ERROR("Failure to bind: %d", ret);
2169 return ret;
2170 }
2171 }
2172 obj_priv->pin_count++;
2173
2174 /* If the object is not active and not pending a flush,
2175 * remove it from the inactive list
2176 */
2177 if (obj_priv->pin_count == 1) {
2178 atomic_inc(&dev->pin_count);
2179 atomic_add(obj->size, &dev->pin_memory);
2180 if (!obj_priv->active &&
2181 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
2182 I915_GEM_DOMAIN_GTT)) == 0 &&
2183 !list_empty(&obj_priv->list))
2184 list_del_init(&obj_priv->list);
2185 }
2186 i915_verify_inactive(dev, __FILE__, __LINE__);
2187
2188 return 0;
2189}
2190
2191void
2192i915_gem_object_unpin(struct drm_gem_object *obj)
2193{
2194 struct drm_device *dev = obj->dev;
2195 drm_i915_private_t *dev_priv = dev->dev_private;
2196 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2197
2198 i915_verify_inactive(dev, __FILE__, __LINE__);
2199 obj_priv->pin_count--;
2200 BUG_ON(obj_priv->pin_count < 0);
2201 BUG_ON(obj_priv->gtt_space == NULL);
2202
2203 /* If the object is no longer pinned, and is
2204 * neither active nor being flushed, then stick it on
2205 * the inactive list
2206 */
2207 if (obj_priv->pin_count == 0) {
2208 if (!obj_priv->active &&
2209 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
2210 I915_GEM_DOMAIN_GTT)) == 0)
2211 list_move_tail(&obj_priv->list,
2212 &dev_priv->mm.inactive_list);
2213 atomic_dec(&dev->pin_count);
2214 atomic_sub(obj->size, &dev->pin_memory);
2215 }
2216 i915_verify_inactive(dev, __FILE__, __LINE__);
2217}
2218
2219int
2220i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2221 struct drm_file *file_priv)
2222{
2223 struct drm_i915_gem_pin *args = data;
2224 struct drm_gem_object *obj;
2225 struct drm_i915_gem_object *obj_priv;
2226 int ret;
2227
2228 mutex_lock(&dev->struct_mutex);
2229
2230 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2231 if (obj == NULL) {
2232 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
2233 args->handle);
2234 mutex_unlock(&dev->struct_mutex);
2235 return -EBADF;
2236 }
2237 obj_priv = obj->driver_private;
2238
2239 ret = i915_gem_object_pin(obj, args->alignment);
2240 if (ret != 0) {
2241 drm_gem_object_unreference(obj);
2242 mutex_unlock(&dev->struct_mutex);
2243 return ret;
2244 }
2245
2246 /* XXX - flush the CPU caches for pinned objects
2247 * as the X server doesn't manage domains yet
2248 */
Eric Anholte47c68e2008-11-14 13:35:19 -08002249 i915_gem_object_flush_cpu_write_domain(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002250 args->offset = obj_priv->gtt_offset;
2251 drm_gem_object_unreference(obj);
2252 mutex_unlock(&dev->struct_mutex);
2253
2254 return 0;
2255}
2256
2257int
2258i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
2259 struct drm_file *file_priv)
2260{
2261 struct drm_i915_gem_pin *args = data;
2262 struct drm_gem_object *obj;
2263
2264 mutex_lock(&dev->struct_mutex);
2265
2266 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2267 if (obj == NULL) {
2268 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
2269 args->handle);
2270 mutex_unlock(&dev->struct_mutex);
2271 return -EBADF;
2272 }
2273
2274 i915_gem_object_unpin(obj);
2275
2276 drm_gem_object_unreference(obj);
2277 mutex_unlock(&dev->struct_mutex);
2278 return 0;
2279}
2280
2281int
2282i915_gem_busy_ioctl(struct drm_device *dev, void *data,
2283 struct drm_file *file_priv)
2284{
2285 struct drm_i915_gem_busy *args = data;
2286 struct drm_gem_object *obj;
2287 struct drm_i915_gem_object *obj_priv;
2288
2289 mutex_lock(&dev->struct_mutex);
2290 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2291 if (obj == NULL) {
2292 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
2293 args->handle);
2294 mutex_unlock(&dev->struct_mutex);
2295 return -EBADF;
2296 }
2297
2298 obj_priv = obj->driver_private;
2299 args->busy = obj_priv->active;
2300
2301 drm_gem_object_unreference(obj);
2302 mutex_unlock(&dev->struct_mutex);
2303 return 0;
2304}
2305
2306int
2307i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
2308 struct drm_file *file_priv)
2309{
2310 return i915_gem_ring_throttle(dev, file_priv);
2311}
2312
2313int i915_gem_init_object(struct drm_gem_object *obj)
2314{
2315 struct drm_i915_gem_object *obj_priv;
2316
2317 obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
2318 if (obj_priv == NULL)
2319 return -ENOMEM;
2320
2321 /*
2322 * We've just allocated pages from the kernel,
2323 * so they've just been written by the CPU with
2324 * zeros. They'll need to be clflushed before we
2325 * use them with the GPU.
2326 */
2327 obj->write_domain = I915_GEM_DOMAIN_CPU;
2328 obj->read_domains = I915_GEM_DOMAIN_CPU;
2329
Keith Packardba1eb1d2008-10-14 19:55:10 -07002330 obj_priv->agp_type = AGP_USER_MEMORY;
2331
Eric Anholt673a3942008-07-30 12:06:12 -07002332 obj->driver_private = obj_priv;
2333 obj_priv->obj = obj;
2334 INIT_LIST_HEAD(&obj_priv->list);
2335 return 0;
2336}
2337
2338void i915_gem_free_object(struct drm_gem_object *obj)
2339{
2340 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2341
2342 while (obj_priv->pin_count > 0)
2343 i915_gem_object_unpin(obj);
2344
2345 i915_gem_object_unbind(obj);
2346
2347 drm_free(obj_priv->page_cpu_valid, 1, DRM_MEM_DRIVER);
2348 drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
2349}
2350
Eric Anholt673a3942008-07-30 12:06:12 -07002351/** Unbinds all objects that are on the given buffer list. */
2352static int
2353i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
2354{
2355 struct drm_gem_object *obj;
2356 struct drm_i915_gem_object *obj_priv;
2357 int ret;
2358
2359 while (!list_empty(head)) {
2360 obj_priv = list_first_entry(head,
2361 struct drm_i915_gem_object,
2362 list);
2363 obj = obj_priv->obj;
2364
2365 if (obj_priv->pin_count != 0) {
2366 DRM_ERROR("Pinned object in unbind list\n");
2367 mutex_unlock(&dev->struct_mutex);
2368 return -EINVAL;
2369 }
2370
2371 ret = i915_gem_object_unbind(obj);
2372 if (ret != 0) {
2373 DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
2374 ret);
2375 mutex_unlock(&dev->struct_mutex);
2376 return ret;
2377 }
2378 }
2379
2380
2381 return 0;
2382}
2383
2384static int
2385i915_gem_idle(struct drm_device *dev)
2386{
2387 drm_i915_private_t *dev_priv = dev->dev_private;
2388 uint32_t seqno, cur_seqno, last_seqno;
2389 int stuck, ret;
2390
Keith Packard6dbe2772008-10-14 21:41:13 -07002391 mutex_lock(&dev->struct_mutex);
2392
2393 if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
2394 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07002395 return 0;
Keith Packard6dbe2772008-10-14 21:41:13 -07002396 }
Eric Anholt673a3942008-07-30 12:06:12 -07002397
2398 /* Hack! Don't let anybody do execbuf while we don't control the chip.
2399 * We need to replace this with a semaphore, or something.
2400 */
2401 dev_priv->mm.suspended = 1;
2402
Keith Packard6dbe2772008-10-14 21:41:13 -07002403 /* Cancel the retire work handler, wait for it to finish if running
2404 */
2405 mutex_unlock(&dev->struct_mutex);
2406 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
2407 mutex_lock(&dev->struct_mutex);
2408
Eric Anholt673a3942008-07-30 12:06:12 -07002409 i915_kernel_lost_context(dev);
2410
2411 /* Flush the GPU along with all non-CPU write domains
2412 */
2413 i915_gem_flush(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT),
2414 ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
2415 seqno = i915_add_request(dev, ~(I915_GEM_DOMAIN_CPU |
2416 I915_GEM_DOMAIN_GTT));
2417
2418 if (seqno == 0) {
2419 mutex_unlock(&dev->struct_mutex);
2420 return -ENOMEM;
2421 }
2422
2423 dev_priv->mm.waiting_gem_seqno = seqno;
2424 last_seqno = 0;
2425 stuck = 0;
2426 for (;;) {
2427 cur_seqno = i915_get_gem_seqno(dev);
2428 if (i915_seqno_passed(cur_seqno, seqno))
2429 break;
2430 if (last_seqno == cur_seqno) {
2431 if (stuck++ > 100) {
2432 DRM_ERROR("hardware wedged\n");
2433 dev_priv->mm.wedged = 1;
2434 DRM_WAKEUP(&dev_priv->irq_queue);
2435 break;
2436 }
2437 }
2438 msleep(10);
2439 last_seqno = cur_seqno;
2440 }
2441 dev_priv->mm.waiting_gem_seqno = 0;
2442
2443 i915_gem_retire_requests(dev);
2444
Eric Anholt28dfe522008-11-13 15:00:55 -08002445 if (!dev_priv->mm.wedged) {
2446 /* Active and flushing should now be empty as we've
2447 * waited for a sequence higher than any pending execbuffer
2448 */
2449 WARN_ON(!list_empty(&dev_priv->mm.active_list));
2450 WARN_ON(!list_empty(&dev_priv->mm.flushing_list));
2451 /* Request should now be empty as we've also waited
2452 * for the last request in the list
2453 */
2454 WARN_ON(!list_empty(&dev_priv->mm.request_list));
2455 }
Eric Anholt673a3942008-07-30 12:06:12 -07002456
Eric Anholt28dfe522008-11-13 15:00:55 -08002457 /* Empty the active and flushing lists to inactive. If there's
2458 * anything left at this point, it means that we're wedged and
2459 * nothing good's going to happen by leaving them there. So strip
2460 * the GPU domains and just stuff them onto inactive.
Eric Anholt673a3942008-07-30 12:06:12 -07002461 */
Eric Anholt28dfe522008-11-13 15:00:55 -08002462 while (!list_empty(&dev_priv->mm.active_list)) {
2463 struct drm_i915_gem_object *obj_priv;
Eric Anholt673a3942008-07-30 12:06:12 -07002464
Eric Anholt28dfe522008-11-13 15:00:55 -08002465 obj_priv = list_first_entry(&dev_priv->mm.active_list,
2466 struct drm_i915_gem_object,
2467 list);
2468 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
2469 i915_gem_object_move_to_inactive(obj_priv->obj);
2470 }
2471
2472 while (!list_empty(&dev_priv->mm.flushing_list)) {
2473 struct drm_i915_gem_object *obj_priv;
2474
Eric Anholt151903d2008-12-01 10:23:21 +10002475 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
Eric Anholt28dfe522008-11-13 15:00:55 -08002476 struct drm_i915_gem_object,
2477 list);
2478 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
2479 i915_gem_object_move_to_inactive(obj_priv->obj);
2480 }
2481
2482
2483 /* Move all inactive buffers out of the GTT. */
Eric Anholt673a3942008-07-30 12:06:12 -07002484 ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
Eric Anholt28dfe522008-11-13 15:00:55 -08002485 WARN_ON(!list_empty(&dev_priv->mm.inactive_list));
Keith Packard6dbe2772008-10-14 21:41:13 -07002486 if (ret) {
2487 mutex_unlock(&dev->struct_mutex);
Eric Anholt673a3942008-07-30 12:06:12 -07002488 return ret;
Keith Packard6dbe2772008-10-14 21:41:13 -07002489 }
Eric Anholt673a3942008-07-30 12:06:12 -07002490
Keith Packard6dbe2772008-10-14 21:41:13 -07002491 i915_gem_cleanup_ringbuffer(dev);
2492 mutex_unlock(&dev->struct_mutex);
2493
Eric Anholt673a3942008-07-30 12:06:12 -07002494 return 0;
2495}
2496
2497static int
2498i915_gem_init_hws(struct drm_device *dev)
2499{
2500 drm_i915_private_t *dev_priv = dev->dev_private;
2501 struct drm_gem_object *obj;
2502 struct drm_i915_gem_object *obj_priv;
2503 int ret;
2504
2505 /* If we need a physical address for the status page, it's already
2506 * initialized at driver load time.
2507 */
2508 if (!I915_NEED_GFX_HWS(dev))
2509 return 0;
2510
2511 obj = drm_gem_object_alloc(dev, 4096);
2512 if (obj == NULL) {
2513 DRM_ERROR("Failed to allocate status page\n");
2514 return -ENOMEM;
2515 }
2516 obj_priv = obj->driver_private;
Keith Packardba1eb1d2008-10-14 19:55:10 -07002517 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
Eric Anholt673a3942008-07-30 12:06:12 -07002518
2519 ret = i915_gem_object_pin(obj, 4096);
2520 if (ret != 0) {
2521 drm_gem_object_unreference(obj);
2522 return ret;
2523 }
2524
2525 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
Eric Anholt673a3942008-07-30 12:06:12 -07002526
Keith Packardba1eb1d2008-10-14 19:55:10 -07002527 dev_priv->hw_status_page = kmap(obj_priv->page_list[0]);
2528 if (dev_priv->hw_status_page == NULL) {
Eric Anholt673a3942008-07-30 12:06:12 -07002529 DRM_ERROR("Failed to map status page.\n");
2530 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2531 drm_gem_object_unreference(obj);
2532 return -EINVAL;
2533 }
2534 dev_priv->hws_obj = obj;
Eric Anholt673a3942008-07-30 12:06:12 -07002535 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
2536 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
Keith Packardba1eb1d2008-10-14 19:55:10 -07002537 I915_READ(HWS_PGA); /* posting read */
Eric Anholt673a3942008-07-30 12:06:12 -07002538 DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
2539
2540 return 0;
2541}
2542
2543static int
2544i915_gem_init_ringbuffer(struct drm_device *dev)
2545{
2546 drm_i915_private_t *dev_priv = dev->dev_private;
2547 struct drm_gem_object *obj;
2548 struct drm_i915_gem_object *obj_priv;
2549 int ret;
Keith Packard50aa253d2008-10-14 17:20:35 -07002550 u32 head;
Eric Anholt673a3942008-07-30 12:06:12 -07002551
2552 ret = i915_gem_init_hws(dev);
2553 if (ret != 0)
2554 return ret;
2555
2556 obj = drm_gem_object_alloc(dev, 128 * 1024);
2557 if (obj == NULL) {
2558 DRM_ERROR("Failed to allocate ringbuffer\n");
2559 return -ENOMEM;
2560 }
2561 obj_priv = obj->driver_private;
2562
2563 ret = i915_gem_object_pin(obj, 4096);
2564 if (ret != 0) {
2565 drm_gem_object_unreference(obj);
2566 return ret;
2567 }
2568
2569 /* Set up the kernel mapping for the ring. */
2570 dev_priv->ring.Size = obj->size;
2571 dev_priv->ring.tail_mask = obj->size - 1;
2572
2573 dev_priv->ring.map.offset = dev->agp->base + obj_priv->gtt_offset;
2574 dev_priv->ring.map.size = obj->size;
2575 dev_priv->ring.map.type = 0;
2576 dev_priv->ring.map.flags = 0;
2577 dev_priv->ring.map.mtrr = 0;
2578
Eric Anholtbd88ee42008-09-23 14:50:57 -07002579 drm_core_ioremap_wc(&dev_priv->ring.map, dev);
Eric Anholt673a3942008-07-30 12:06:12 -07002580 if (dev_priv->ring.map.handle == NULL) {
2581 DRM_ERROR("Failed to map ringbuffer.\n");
2582 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2583 drm_gem_object_unreference(obj);
2584 return -EINVAL;
2585 }
2586 dev_priv->ring.ring_obj = obj;
2587 dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
2588
2589 /* Stop the ring if it's running. */
2590 I915_WRITE(PRB0_CTL, 0);
Eric Anholt673a3942008-07-30 12:06:12 -07002591 I915_WRITE(PRB0_TAIL, 0);
Keith Packard50aa253d2008-10-14 17:20:35 -07002592 I915_WRITE(PRB0_HEAD, 0);
Eric Anholt673a3942008-07-30 12:06:12 -07002593
2594 /* Initialize the ring. */
2595 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
Keith Packard50aa253d2008-10-14 17:20:35 -07002596 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
2597
2598 /* G45 ring initialization fails to reset head to zero */
2599 if (head != 0) {
2600 DRM_ERROR("Ring head not reset to zero "
2601 "ctl %08x head %08x tail %08x start %08x\n",
2602 I915_READ(PRB0_CTL),
2603 I915_READ(PRB0_HEAD),
2604 I915_READ(PRB0_TAIL),
2605 I915_READ(PRB0_START));
2606 I915_WRITE(PRB0_HEAD, 0);
2607
2608 DRM_ERROR("Ring head forced to zero "
2609 "ctl %08x head %08x tail %08x start %08x\n",
2610 I915_READ(PRB0_CTL),
2611 I915_READ(PRB0_HEAD),
2612 I915_READ(PRB0_TAIL),
2613 I915_READ(PRB0_START));
2614 }
2615
Eric Anholt673a3942008-07-30 12:06:12 -07002616 I915_WRITE(PRB0_CTL,
2617 ((obj->size - 4096) & RING_NR_PAGES) |
2618 RING_NO_REPORT |
2619 RING_VALID);
2620
Keith Packard50aa253d2008-10-14 17:20:35 -07002621 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
2622
2623 /* If the head is still not zero, the ring is dead */
2624 if (head != 0) {
2625 DRM_ERROR("Ring initialization failed "
2626 "ctl %08x head %08x tail %08x start %08x\n",
2627 I915_READ(PRB0_CTL),
2628 I915_READ(PRB0_HEAD),
2629 I915_READ(PRB0_TAIL),
2630 I915_READ(PRB0_START));
2631 return -EIO;
2632 }
2633
Eric Anholt673a3942008-07-30 12:06:12 -07002634 /* Update our cache of the ring state */
2635 i915_kernel_lost_context(dev);
2636
2637 return 0;
2638}
2639
2640static void
2641i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2642{
2643 drm_i915_private_t *dev_priv = dev->dev_private;
2644
2645 if (dev_priv->ring.ring_obj == NULL)
2646 return;
2647
2648 drm_core_ioremapfree(&dev_priv->ring.map, dev);
2649
2650 i915_gem_object_unpin(dev_priv->ring.ring_obj);
2651 drm_gem_object_unreference(dev_priv->ring.ring_obj);
2652 dev_priv->ring.ring_obj = NULL;
2653 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2654
2655 if (dev_priv->hws_obj != NULL) {
Keith Packardba1eb1d2008-10-14 19:55:10 -07002656 struct drm_gem_object *obj = dev_priv->hws_obj;
2657 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2658
2659 kunmap(obj_priv->page_list[0]);
2660 i915_gem_object_unpin(obj);
2661 drm_gem_object_unreference(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07002662 dev_priv->hws_obj = NULL;
2663 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
Keith Packardba1eb1d2008-10-14 19:55:10 -07002664 dev_priv->hw_status_page = NULL;
Eric Anholt673a3942008-07-30 12:06:12 -07002665
2666 /* Write high address into HWS_PGA when disabling. */
2667 I915_WRITE(HWS_PGA, 0x1ffff000);
2668 }
2669}
2670
2671int
2672i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2673 struct drm_file *file_priv)
2674{
2675 drm_i915_private_t *dev_priv = dev->dev_private;
2676 int ret;
2677
2678 if (dev_priv->mm.wedged) {
2679 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2680 dev_priv->mm.wedged = 0;
2681 }
2682
2683 ret = i915_gem_init_ringbuffer(dev);
2684 if (ret != 0)
2685 return ret;
2686
Keith Packard0839ccb2008-10-30 19:38:48 -07002687 dev_priv->mm.gtt_mapping = io_mapping_create_wc(dev->agp->base,
2688 dev->agp->agp_info.aper_size
2689 * 1024 * 1024);
2690
Eric Anholt673a3942008-07-30 12:06:12 -07002691 mutex_lock(&dev->struct_mutex);
2692 BUG_ON(!list_empty(&dev_priv->mm.active_list));
2693 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2694 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
2695 BUG_ON(!list_empty(&dev_priv->mm.request_list));
2696 dev_priv->mm.suspended = 0;
2697 mutex_unlock(&dev->struct_mutex);
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04002698
2699 drm_irq_install(dev);
2700
Eric Anholt673a3942008-07-30 12:06:12 -07002701 return 0;
2702}
2703
2704int
2705i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
2706 struct drm_file *file_priv)
2707{
Keith Packard0839ccb2008-10-30 19:38:48 -07002708 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt673a3942008-07-30 12:06:12 -07002709 int ret;
2710
Eric Anholt673a3942008-07-30 12:06:12 -07002711 ret = i915_gem_idle(dev);
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04002712 drm_irq_uninstall(dev);
2713
Keith Packard0839ccb2008-10-30 19:38:48 -07002714 io_mapping_free(dev_priv->mm.gtt_mapping);
Keith Packard6dbe2772008-10-14 21:41:13 -07002715 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002716}
2717
2718void
2719i915_gem_lastclose(struct drm_device *dev)
2720{
2721 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002722
Keith Packard6dbe2772008-10-14 21:41:13 -07002723 ret = i915_gem_idle(dev);
2724 if (ret)
2725 DRM_ERROR("failed to idle hardware: %d\n", ret);
Eric Anholt673a3942008-07-30 12:06:12 -07002726}
2727
2728void
2729i915_gem_load(struct drm_device *dev)
2730{
2731 drm_i915_private_t *dev_priv = dev->dev_private;
2732
2733 INIT_LIST_HEAD(&dev_priv->mm.active_list);
2734 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
2735 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
2736 INIT_LIST_HEAD(&dev_priv->mm.request_list);
2737 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
2738 i915_gem_retire_work_handler);
Eric Anholt673a3942008-07-30 12:06:12 -07002739 dev_priv->mm.next_gem_seqno = 1;
2740
2741 i915_gem_detect_bit_6_swizzle(dev);
2742}