blob: 5fe50343de11910ba30f64477c52ac31710c0b63 [file] [log] [blame]
Eric Anholt673a3942008-07-30 12:06:12 -07001/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
32#include <linux/swap.h>
33
34static int
35i915_gem_object_set_domain(struct drm_gem_object *obj,
36 uint32_t read_domains,
37 uint32_t write_domain);
38static int
39i915_gem_object_set_domain_range(struct drm_gem_object *obj,
40 uint64_t offset,
41 uint64_t size,
42 uint32_t read_domains,
43 uint32_t write_domain);
44static int
45i915_gem_set_domain(struct drm_gem_object *obj,
46 struct drm_file *file_priv,
47 uint32_t read_domains,
48 uint32_t write_domain);
49static int i915_gem_object_get_page_list(struct drm_gem_object *obj);
50static void i915_gem_object_free_page_list(struct drm_gem_object *obj);
51static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
52
53int
54i915_gem_init_ioctl(struct drm_device *dev, void *data,
55 struct drm_file *file_priv)
56{
57 drm_i915_private_t *dev_priv = dev->dev_private;
58 struct drm_i915_gem_init *args = data;
59
60 mutex_lock(&dev->struct_mutex);
61
62 if (args->gtt_start >= args->gtt_end ||
63 (args->gtt_start & (PAGE_SIZE - 1)) != 0 ||
64 (args->gtt_end & (PAGE_SIZE - 1)) != 0) {
65 mutex_unlock(&dev->struct_mutex);
66 return -EINVAL;
67 }
68
69 drm_mm_init(&dev_priv->mm.gtt_space, args->gtt_start,
70 args->gtt_end - args->gtt_start);
71
72 dev->gtt_total = (uint32_t) (args->gtt_end - args->gtt_start);
73
74 mutex_unlock(&dev->struct_mutex);
75
76 return 0;
77}
78
79
80/**
81 * Creates a new mm object and returns a handle to it.
82 */
83int
84i915_gem_create_ioctl(struct drm_device *dev, void *data,
85 struct drm_file *file_priv)
86{
87 struct drm_i915_gem_create *args = data;
88 struct drm_gem_object *obj;
89 int handle, ret;
90
91 args->size = roundup(args->size, PAGE_SIZE);
92
93 /* Allocate the new object */
94 obj = drm_gem_object_alloc(dev, args->size);
95 if (obj == NULL)
96 return -ENOMEM;
97
98 ret = drm_gem_handle_create(file_priv, obj, &handle);
99 mutex_lock(&dev->struct_mutex);
100 drm_gem_object_handle_unreference(obj);
101 mutex_unlock(&dev->struct_mutex);
102
103 if (ret)
104 return ret;
105
106 args->handle = handle;
107
108 return 0;
109}
110
111/**
112 * Reads data from the object referenced by handle.
113 *
114 * On error, the contents of *data are undefined.
115 */
116int
117i915_gem_pread_ioctl(struct drm_device *dev, void *data,
118 struct drm_file *file_priv)
119{
120 struct drm_i915_gem_pread *args = data;
121 struct drm_gem_object *obj;
122 struct drm_i915_gem_object *obj_priv;
123 ssize_t read;
124 loff_t offset;
125 int ret;
126
127 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
128 if (obj == NULL)
129 return -EBADF;
130 obj_priv = obj->driver_private;
131
132 /* Bounds check source.
133 *
134 * XXX: This could use review for overflow issues...
135 */
136 if (args->offset > obj->size || args->size > obj->size ||
137 args->offset + args->size > obj->size) {
138 drm_gem_object_unreference(obj);
139 return -EINVAL;
140 }
141
142 mutex_lock(&dev->struct_mutex);
143
144 ret = i915_gem_object_set_domain_range(obj, args->offset, args->size,
145 I915_GEM_DOMAIN_CPU, 0);
146 if (ret != 0) {
147 drm_gem_object_unreference(obj);
148 mutex_unlock(&dev->struct_mutex);
149 }
150
151 offset = args->offset;
152
153 read = vfs_read(obj->filp, (char __user *)(uintptr_t)args->data_ptr,
154 args->size, &offset);
155 if (read != args->size) {
156 drm_gem_object_unreference(obj);
157 mutex_unlock(&dev->struct_mutex);
158 if (read < 0)
159 return read;
160 else
161 return -EINVAL;
162 }
163
164 drm_gem_object_unreference(obj);
165 mutex_unlock(&dev->struct_mutex);
166
167 return 0;
168}
169
170static int
171i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
172 struct drm_i915_gem_pwrite *args,
173 struct drm_file *file_priv)
174{
175 struct drm_i915_gem_object *obj_priv = obj->driver_private;
176 ssize_t remain;
177 loff_t offset;
178 char __user *user_data;
179 char *vaddr;
180 int i, o, l;
181 int ret = 0;
182 unsigned long pfn;
183 unsigned long unwritten;
184
185 user_data = (char __user *) (uintptr_t) args->data_ptr;
186 remain = args->size;
187 if (!access_ok(VERIFY_READ, user_data, remain))
188 return -EFAULT;
189
190
191 mutex_lock(&dev->struct_mutex);
192 ret = i915_gem_object_pin(obj, 0);
193 if (ret) {
194 mutex_unlock(&dev->struct_mutex);
195 return ret;
196 }
197 ret = i915_gem_set_domain(obj, file_priv,
198 I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
199 if (ret)
200 goto fail;
201
202 obj_priv = obj->driver_private;
203 offset = obj_priv->gtt_offset + args->offset;
204 obj_priv->dirty = 1;
205
206 while (remain > 0) {
207 /* Operation in this page
208 *
209 * i = page number
210 * o = offset within page
211 * l = bytes to copy
212 */
213 i = offset >> PAGE_SHIFT;
214 o = offset & (PAGE_SIZE-1);
215 l = remain;
216 if ((o + l) > PAGE_SIZE)
217 l = PAGE_SIZE - o;
218
219 pfn = (dev->agp->base >> PAGE_SHIFT) + i;
220
221#ifdef CONFIG_HIGHMEM
222 /* kmap_atomic can't map IO pages on non-HIGHMEM kernels
223 */
224 vaddr = kmap_atomic_pfn(pfn, KM_USER0);
225#if WATCH_PWRITE
226 DRM_INFO("pwrite i %d o %d l %d pfn %ld vaddr %p\n",
227 i, o, l, pfn, vaddr);
228#endif
229 unwritten = __copy_from_user_inatomic_nocache(vaddr + o,
230 user_data, l);
231 kunmap_atomic(vaddr, KM_USER0);
232
233 if (unwritten)
234#endif /* CONFIG_HIGHMEM */
235 {
236 vaddr = ioremap(pfn << PAGE_SHIFT, PAGE_SIZE);
237#if WATCH_PWRITE
238 DRM_INFO("pwrite slow i %d o %d l %d "
239 "pfn %ld vaddr %p\n",
240 i, o, l, pfn, vaddr);
241#endif
242 if (vaddr == NULL) {
243 ret = -EFAULT;
244 goto fail;
245 }
246 unwritten = __copy_from_user(vaddr + o, user_data, l);
247#if WATCH_PWRITE
248 DRM_INFO("unwritten %ld\n", unwritten);
249#endif
250 iounmap(vaddr);
251 if (unwritten) {
252 ret = -EFAULT;
253 goto fail;
254 }
255 }
256
257 remain -= l;
258 user_data += l;
259 offset += l;
260 }
261#if WATCH_PWRITE && 1
262 i915_gem_clflush_object(obj);
263 i915_gem_dump_object(obj, args->offset + args->size, __func__, ~0);
264 i915_gem_clflush_object(obj);
265#endif
266
267fail:
268 i915_gem_object_unpin(obj);
269 mutex_unlock(&dev->struct_mutex);
270
271 return ret;
272}
273
274int
275i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
276 struct drm_i915_gem_pwrite *args,
277 struct drm_file *file_priv)
278{
279 int ret;
280 loff_t offset;
281 ssize_t written;
282
283 mutex_lock(&dev->struct_mutex);
284
285 ret = i915_gem_set_domain(obj, file_priv,
286 I915_GEM_DOMAIN_CPU, I915_GEM_DOMAIN_CPU);
287 if (ret) {
288 mutex_unlock(&dev->struct_mutex);
289 return ret;
290 }
291
292 offset = args->offset;
293
294 written = vfs_write(obj->filp,
295 (char __user *)(uintptr_t) args->data_ptr,
296 args->size, &offset);
297 if (written != args->size) {
298 mutex_unlock(&dev->struct_mutex);
299 if (written < 0)
300 return written;
301 else
302 return -EINVAL;
303 }
304
305 mutex_unlock(&dev->struct_mutex);
306
307 return 0;
308}
309
310/**
311 * Writes data to the object referenced by handle.
312 *
313 * On error, the contents of the buffer that were to be modified are undefined.
314 */
315int
316i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
317 struct drm_file *file_priv)
318{
319 struct drm_i915_gem_pwrite *args = data;
320 struct drm_gem_object *obj;
321 struct drm_i915_gem_object *obj_priv;
322 int ret = 0;
323
324 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
325 if (obj == NULL)
326 return -EBADF;
327 obj_priv = obj->driver_private;
328
329 /* Bounds check destination.
330 *
331 * XXX: This could use review for overflow issues...
332 */
333 if (args->offset > obj->size || args->size > obj->size ||
334 args->offset + args->size > obj->size) {
335 drm_gem_object_unreference(obj);
336 return -EINVAL;
337 }
338
339 /* We can only do the GTT pwrite on untiled buffers, as otherwise
340 * it would end up going through the fenced access, and we'll get
341 * different detiling behavior between reading and writing.
342 * pread/pwrite currently are reading and writing from the CPU
343 * perspective, requiring manual detiling by the client.
344 */
345 if (obj_priv->tiling_mode == I915_TILING_NONE &&
346 dev->gtt_total != 0)
347 ret = i915_gem_gtt_pwrite(dev, obj, args, file_priv);
348 else
349 ret = i915_gem_shmem_pwrite(dev, obj, args, file_priv);
350
351#if WATCH_PWRITE
352 if (ret)
353 DRM_INFO("pwrite failed %d\n", ret);
354#endif
355
356 drm_gem_object_unreference(obj);
357
358 return ret;
359}
360
361/**
362 * Called when user space prepares to use an object
363 */
364int
365i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
366 struct drm_file *file_priv)
367{
368 struct drm_i915_gem_set_domain *args = data;
369 struct drm_gem_object *obj;
370 int ret;
371
372 if (!(dev->driver->driver_features & DRIVER_GEM))
373 return -ENODEV;
374
375 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
376 if (obj == NULL)
377 return -EBADF;
378
379 mutex_lock(&dev->struct_mutex);
380#if WATCH_BUF
381 DRM_INFO("set_domain_ioctl %p(%d), %08x %08x\n",
382 obj, obj->size, args->read_domains, args->write_domain);
383#endif
384 ret = i915_gem_set_domain(obj, file_priv,
385 args->read_domains, args->write_domain);
386 drm_gem_object_unreference(obj);
387 mutex_unlock(&dev->struct_mutex);
388 return ret;
389}
390
391/**
392 * Called when user space has done writes to this buffer
393 */
394int
395i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
396 struct drm_file *file_priv)
397{
398 struct drm_i915_gem_sw_finish *args = data;
399 struct drm_gem_object *obj;
400 struct drm_i915_gem_object *obj_priv;
401 int ret = 0;
402
403 if (!(dev->driver->driver_features & DRIVER_GEM))
404 return -ENODEV;
405
406 mutex_lock(&dev->struct_mutex);
407 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
408 if (obj == NULL) {
409 mutex_unlock(&dev->struct_mutex);
410 return -EBADF;
411 }
412
413#if WATCH_BUF
414 DRM_INFO("%s: sw_finish %d (%p %d)\n",
415 __func__, args->handle, obj, obj->size);
416#endif
417 obj_priv = obj->driver_private;
418
419 /* Pinned buffers may be scanout, so flush the cache */
420 if ((obj->write_domain & I915_GEM_DOMAIN_CPU) && obj_priv->pin_count) {
421 i915_gem_clflush_object(obj);
422 drm_agp_chipset_flush(dev);
423 }
424 drm_gem_object_unreference(obj);
425 mutex_unlock(&dev->struct_mutex);
426 return ret;
427}
428
429/**
430 * Maps the contents of an object, returning the address it is mapped
431 * into.
432 *
433 * While the mapping holds a reference on the contents of the object, it doesn't
434 * imply a ref on the object itself.
435 */
436int
437i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
438 struct drm_file *file_priv)
439{
440 struct drm_i915_gem_mmap *args = data;
441 struct drm_gem_object *obj;
442 loff_t offset;
443 unsigned long addr;
444
445 if (!(dev->driver->driver_features & DRIVER_GEM))
446 return -ENODEV;
447
448 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
449 if (obj == NULL)
450 return -EBADF;
451
452 offset = args->offset;
453
454 down_write(&current->mm->mmap_sem);
455 addr = do_mmap(obj->filp, 0, args->size,
456 PROT_READ | PROT_WRITE, MAP_SHARED,
457 args->offset);
458 up_write(&current->mm->mmap_sem);
459 mutex_lock(&dev->struct_mutex);
460 drm_gem_object_unreference(obj);
461 mutex_unlock(&dev->struct_mutex);
462 if (IS_ERR((void *)addr))
463 return addr;
464
465 args->addr_ptr = (uint64_t) addr;
466
467 return 0;
468}
469
470static void
471i915_gem_object_free_page_list(struct drm_gem_object *obj)
472{
473 struct drm_i915_gem_object *obj_priv = obj->driver_private;
474 int page_count = obj->size / PAGE_SIZE;
475 int i;
476
477 if (obj_priv->page_list == NULL)
478 return;
479
480
481 for (i = 0; i < page_count; i++)
482 if (obj_priv->page_list[i] != NULL) {
483 if (obj_priv->dirty)
484 set_page_dirty(obj_priv->page_list[i]);
485 mark_page_accessed(obj_priv->page_list[i]);
486 page_cache_release(obj_priv->page_list[i]);
487 }
488 obj_priv->dirty = 0;
489
490 drm_free(obj_priv->page_list,
491 page_count * sizeof(struct page *),
492 DRM_MEM_DRIVER);
493 obj_priv->page_list = NULL;
494}
495
496static void
497i915_gem_object_move_to_active(struct drm_gem_object *obj)
498{
499 struct drm_device *dev = obj->dev;
500 drm_i915_private_t *dev_priv = dev->dev_private;
501 struct drm_i915_gem_object *obj_priv = obj->driver_private;
502
503 /* Add a reference if we're newly entering the active list. */
504 if (!obj_priv->active) {
505 drm_gem_object_reference(obj);
506 obj_priv->active = 1;
507 }
508 /* Move from whatever list we were on to the tail of execution. */
509 list_move_tail(&obj_priv->list,
510 &dev_priv->mm.active_list);
511}
512
513
514static void
515i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
516{
517 struct drm_device *dev = obj->dev;
518 drm_i915_private_t *dev_priv = dev->dev_private;
519 struct drm_i915_gem_object *obj_priv = obj->driver_private;
520
521 i915_verify_inactive(dev, __FILE__, __LINE__);
522 if (obj_priv->pin_count != 0)
523 list_del_init(&obj_priv->list);
524 else
525 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list);
526
527 if (obj_priv->active) {
528 obj_priv->active = 0;
529 drm_gem_object_unreference(obj);
530 }
531 i915_verify_inactive(dev, __FILE__, __LINE__);
532}
533
534/**
535 * Creates a new sequence number, emitting a write of it to the status page
536 * plus an interrupt, which will trigger i915_user_interrupt_handler.
537 *
538 * Must be called with struct_lock held.
539 *
540 * Returned sequence numbers are nonzero on success.
541 */
542static uint32_t
543i915_add_request(struct drm_device *dev, uint32_t flush_domains)
544{
545 drm_i915_private_t *dev_priv = dev->dev_private;
546 struct drm_i915_gem_request *request;
547 uint32_t seqno;
548 int was_empty;
549 RING_LOCALS;
550
551 request = drm_calloc(1, sizeof(*request), DRM_MEM_DRIVER);
552 if (request == NULL)
553 return 0;
554
555 /* Grab the seqno we're going to make this request be, and bump the
556 * next (skipping 0 so it can be the reserved no-seqno value).
557 */
558 seqno = dev_priv->mm.next_gem_seqno;
559 dev_priv->mm.next_gem_seqno++;
560 if (dev_priv->mm.next_gem_seqno == 0)
561 dev_priv->mm.next_gem_seqno++;
562
563 BEGIN_LP_RING(4);
564 OUT_RING(MI_STORE_DWORD_INDEX);
565 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
566 OUT_RING(seqno);
567
568 OUT_RING(MI_USER_INTERRUPT);
569 ADVANCE_LP_RING();
570
571 DRM_DEBUG("%d\n", seqno);
572
573 request->seqno = seqno;
574 request->emitted_jiffies = jiffies;
575 request->flush_domains = flush_domains;
576 was_empty = list_empty(&dev_priv->mm.request_list);
577 list_add_tail(&request->list, &dev_priv->mm.request_list);
578
579 if (was_empty)
580 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
581 return seqno;
582}
583
584/**
585 * Command execution barrier
586 *
587 * Ensures that all commands in the ring are finished
588 * before signalling the CPU
589 */
590uint32_t
591i915_retire_commands(struct drm_device *dev)
592{
593 drm_i915_private_t *dev_priv = dev->dev_private;
594 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
595 uint32_t flush_domains = 0;
596 RING_LOCALS;
597
598 /* The sampler always gets flushed on i965 (sigh) */
599 if (IS_I965G(dev))
600 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
601 BEGIN_LP_RING(2);
602 OUT_RING(cmd);
603 OUT_RING(0); /* noop */
604 ADVANCE_LP_RING();
605 return flush_domains;
606}
607
608/**
609 * Moves buffers associated only with the given active seqno from the active
610 * to inactive list, potentially freeing them.
611 */
612static void
613i915_gem_retire_request(struct drm_device *dev,
614 struct drm_i915_gem_request *request)
615{
616 drm_i915_private_t *dev_priv = dev->dev_private;
617
618 /* Move any buffers on the active list that are no longer referenced
619 * by the ringbuffer to the flushing/inactive lists as appropriate.
620 */
621 while (!list_empty(&dev_priv->mm.active_list)) {
622 struct drm_gem_object *obj;
623 struct drm_i915_gem_object *obj_priv;
624
625 obj_priv = list_first_entry(&dev_priv->mm.active_list,
626 struct drm_i915_gem_object,
627 list);
628 obj = obj_priv->obj;
629
630 /* If the seqno being retired doesn't match the oldest in the
631 * list, then the oldest in the list must still be newer than
632 * this seqno.
633 */
634 if (obj_priv->last_rendering_seqno != request->seqno)
635 return;
636#if WATCH_LRU
637 DRM_INFO("%s: retire %d moves to inactive list %p\n",
638 __func__, request->seqno, obj);
639#endif
640
641 if (obj->write_domain != 0) {
642 list_move_tail(&obj_priv->list,
643 &dev_priv->mm.flushing_list);
644 } else {
645 i915_gem_object_move_to_inactive(obj);
646 }
647 }
648
649 if (request->flush_domains != 0) {
650 struct drm_i915_gem_object *obj_priv, *next;
651
652 /* Clear the write domain and activity from any buffers
653 * that are just waiting for a flush matching the one retired.
654 */
655 list_for_each_entry_safe(obj_priv, next,
656 &dev_priv->mm.flushing_list, list) {
657 struct drm_gem_object *obj = obj_priv->obj;
658
659 if (obj->write_domain & request->flush_domains) {
660 obj->write_domain = 0;
661 i915_gem_object_move_to_inactive(obj);
662 }
663 }
664
665 }
666}
667
668/**
669 * Returns true if seq1 is later than seq2.
670 */
671static int
672i915_seqno_passed(uint32_t seq1, uint32_t seq2)
673{
674 return (int32_t)(seq1 - seq2) >= 0;
675}
676
677uint32_t
678i915_get_gem_seqno(struct drm_device *dev)
679{
680 drm_i915_private_t *dev_priv = dev->dev_private;
681
682 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
683}
684
685/**
686 * This function clears the request list as sequence numbers are passed.
687 */
688void
689i915_gem_retire_requests(struct drm_device *dev)
690{
691 drm_i915_private_t *dev_priv = dev->dev_private;
692 uint32_t seqno;
693
694 seqno = i915_get_gem_seqno(dev);
695
696 while (!list_empty(&dev_priv->mm.request_list)) {
697 struct drm_i915_gem_request *request;
698 uint32_t retiring_seqno;
699
700 request = list_first_entry(&dev_priv->mm.request_list,
701 struct drm_i915_gem_request,
702 list);
703 retiring_seqno = request->seqno;
704
705 if (i915_seqno_passed(seqno, retiring_seqno) ||
706 dev_priv->mm.wedged) {
707 i915_gem_retire_request(dev, request);
708
709 list_del(&request->list);
710 drm_free(request, sizeof(*request), DRM_MEM_DRIVER);
711 } else
712 break;
713 }
714}
715
716void
717i915_gem_retire_work_handler(struct work_struct *work)
718{
719 drm_i915_private_t *dev_priv;
720 struct drm_device *dev;
721
722 dev_priv = container_of(work, drm_i915_private_t,
723 mm.retire_work.work);
724 dev = dev_priv->dev;
725
726 mutex_lock(&dev->struct_mutex);
727 i915_gem_retire_requests(dev);
728 if (!list_empty(&dev_priv->mm.request_list))
729 schedule_delayed_work(&dev_priv->mm.retire_work, HZ);
730 mutex_unlock(&dev->struct_mutex);
731}
732
733/**
734 * Waits for a sequence number to be signaled, and cleans up the
735 * request and object lists appropriately for that event.
736 */
737int
738i915_wait_request(struct drm_device *dev, uint32_t seqno)
739{
740 drm_i915_private_t *dev_priv = dev->dev_private;
741 int ret = 0;
742
743 BUG_ON(seqno == 0);
744
745 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
746 dev_priv->mm.waiting_gem_seqno = seqno;
747 i915_user_irq_get(dev);
748 ret = wait_event_interruptible(dev_priv->irq_queue,
749 i915_seqno_passed(i915_get_gem_seqno(dev),
750 seqno) ||
751 dev_priv->mm.wedged);
752 i915_user_irq_put(dev);
753 dev_priv->mm.waiting_gem_seqno = 0;
754 }
755 if (dev_priv->mm.wedged)
756 ret = -EIO;
757
758 if (ret && ret != -ERESTARTSYS)
759 DRM_ERROR("%s returns %d (awaiting %d at %d)\n",
760 __func__, ret, seqno, i915_get_gem_seqno(dev));
761
762 /* Directly dispatch request retiring. While we have the work queue
763 * to handle this, the waiter on a request often wants an associated
764 * buffer to have made it to the inactive list, and we would need
765 * a separate wait queue to handle that.
766 */
767 if (ret == 0)
768 i915_gem_retire_requests(dev);
769
770 return ret;
771}
772
773static void
774i915_gem_flush(struct drm_device *dev,
775 uint32_t invalidate_domains,
776 uint32_t flush_domains)
777{
778 drm_i915_private_t *dev_priv = dev->dev_private;
779 uint32_t cmd;
780 RING_LOCALS;
781
782#if WATCH_EXEC
783 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
784 invalidate_domains, flush_domains);
785#endif
786
787 if (flush_domains & I915_GEM_DOMAIN_CPU)
788 drm_agp_chipset_flush(dev);
789
790 if ((invalidate_domains | flush_domains) & ~(I915_GEM_DOMAIN_CPU |
791 I915_GEM_DOMAIN_GTT)) {
792 /*
793 * read/write caches:
794 *
795 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
796 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
797 * also flushed at 2d versus 3d pipeline switches.
798 *
799 * read-only caches:
800 *
801 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
802 * MI_READ_FLUSH is set, and is always flushed on 965.
803 *
804 * I915_GEM_DOMAIN_COMMAND may not exist?
805 *
806 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
807 * invalidated when MI_EXE_FLUSH is set.
808 *
809 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
810 * invalidated with every MI_FLUSH.
811 *
812 * TLBs:
813 *
814 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
815 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
816 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
817 * are flushed at any MI_FLUSH.
818 */
819
820 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
821 if ((invalidate_domains|flush_domains) &
822 I915_GEM_DOMAIN_RENDER)
823 cmd &= ~MI_NO_WRITE_FLUSH;
824 if (!IS_I965G(dev)) {
825 /*
826 * On the 965, the sampler cache always gets flushed
827 * and this bit is reserved.
828 */
829 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
830 cmd |= MI_READ_FLUSH;
831 }
832 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
833 cmd |= MI_EXE_FLUSH;
834
835#if WATCH_EXEC
836 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
837#endif
838 BEGIN_LP_RING(2);
839 OUT_RING(cmd);
840 OUT_RING(0); /* noop */
841 ADVANCE_LP_RING();
842 }
843}
844
845/**
846 * Ensures that all rendering to the object has completed and the object is
847 * safe to unbind from the GTT or access from the CPU.
848 */
849static int
850i915_gem_object_wait_rendering(struct drm_gem_object *obj)
851{
852 struct drm_device *dev = obj->dev;
853 struct drm_i915_gem_object *obj_priv = obj->driver_private;
854 int ret;
855
856 /* If there are writes queued to the buffer, flush and
857 * create a new seqno to wait for.
858 */
859 if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)) {
860 uint32_t write_domain = obj->write_domain;
861#if WATCH_BUF
862 DRM_INFO("%s: flushing object %p from write domain %08x\n",
863 __func__, obj, write_domain);
864#endif
865 i915_gem_flush(dev, 0, write_domain);
866
867 i915_gem_object_move_to_active(obj);
868 obj_priv->last_rendering_seqno = i915_add_request(dev,
869 write_domain);
870 BUG_ON(obj_priv->last_rendering_seqno == 0);
871#if WATCH_LRU
872 DRM_INFO("%s: flush moves to exec list %p\n", __func__, obj);
873#endif
874 }
875
876 /* If there is rendering queued on the buffer being evicted, wait for
877 * it.
878 */
879 if (obj_priv->active) {
880#if WATCH_BUF
881 DRM_INFO("%s: object %p wait for seqno %08x\n",
882 __func__, obj, obj_priv->last_rendering_seqno);
883#endif
884 ret = i915_wait_request(dev, obj_priv->last_rendering_seqno);
885 if (ret != 0)
886 return ret;
887 }
888
889 return 0;
890}
891
892/**
893 * Unbinds an object from the GTT aperture.
894 */
895static int
896i915_gem_object_unbind(struct drm_gem_object *obj)
897{
898 struct drm_device *dev = obj->dev;
899 struct drm_i915_gem_object *obj_priv = obj->driver_private;
900 int ret = 0;
901
902#if WATCH_BUF
903 DRM_INFO("%s:%d %p\n", __func__, __LINE__, obj);
904 DRM_INFO("gtt_space %p\n", obj_priv->gtt_space);
905#endif
906 if (obj_priv->gtt_space == NULL)
907 return 0;
908
909 if (obj_priv->pin_count != 0) {
910 DRM_ERROR("Attempting to unbind pinned buffer\n");
911 return -EINVAL;
912 }
913
914 /* Wait for any rendering to complete
915 */
916 ret = i915_gem_object_wait_rendering(obj);
917 if (ret) {
918 DRM_ERROR("wait_rendering failed: %d\n", ret);
919 return ret;
920 }
921
922 /* Move the object to the CPU domain to ensure that
923 * any possible CPU writes while it's not in the GTT
924 * are flushed when we go to remap it. This will
925 * also ensure that all pending GPU writes are finished
926 * before we unbind.
927 */
928 ret = i915_gem_object_set_domain(obj, I915_GEM_DOMAIN_CPU,
929 I915_GEM_DOMAIN_CPU);
930 if (ret) {
931 DRM_ERROR("set_domain failed: %d\n", ret);
932 return ret;
933 }
934
935 if (obj_priv->agp_mem != NULL) {
936 drm_unbind_agp(obj_priv->agp_mem);
937 drm_free_agp(obj_priv->agp_mem, obj->size / PAGE_SIZE);
938 obj_priv->agp_mem = NULL;
939 }
940
941 BUG_ON(obj_priv->active);
942
943 i915_gem_object_free_page_list(obj);
944
945 if (obj_priv->gtt_space) {
946 atomic_dec(&dev->gtt_count);
947 atomic_sub(obj->size, &dev->gtt_memory);
948
949 drm_mm_put_block(obj_priv->gtt_space);
950 obj_priv->gtt_space = NULL;
951 }
952
953 /* Remove ourselves from the LRU list if present. */
954 if (!list_empty(&obj_priv->list))
955 list_del_init(&obj_priv->list);
956
957 return 0;
958}
959
960static int
961i915_gem_evict_something(struct drm_device *dev)
962{
963 drm_i915_private_t *dev_priv = dev->dev_private;
964 struct drm_gem_object *obj;
965 struct drm_i915_gem_object *obj_priv;
966 int ret = 0;
967
968 for (;;) {
969 /* If there's an inactive buffer available now, grab it
970 * and be done.
971 */
972 if (!list_empty(&dev_priv->mm.inactive_list)) {
973 obj_priv = list_first_entry(&dev_priv->mm.inactive_list,
974 struct drm_i915_gem_object,
975 list);
976 obj = obj_priv->obj;
977 BUG_ON(obj_priv->pin_count != 0);
978#if WATCH_LRU
979 DRM_INFO("%s: evicting %p\n", __func__, obj);
980#endif
981 BUG_ON(obj_priv->active);
982
983 /* Wait on the rendering and unbind the buffer. */
984 ret = i915_gem_object_unbind(obj);
985 break;
986 }
987
988 /* If we didn't get anything, but the ring is still processing
989 * things, wait for one of those things to finish and hopefully
990 * leave us a buffer to evict.
991 */
992 if (!list_empty(&dev_priv->mm.request_list)) {
993 struct drm_i915_gem_request *request;
994
995 request = list_first_entry(&dev_priv->mm.request_list,
996 struct drm_i915_gem_request,
997 list);
998
999 ret = i915_wait_request(dev, request->seqno);
1000 if (ret)
1001 break;
1002
1003 /* if waiting caused an object to become inactive,
1004 * then loop around and wait for it. Otherwise, we
1005 * assume that waiting freed and unbound something,
1006 * so there should now be some space in the GTT
1007 */
1008 if (!list_empty(&dev_priv->mm.inactive_list))
1009 continue;
1010 break;
1011 }
1012
1013 /* If we didn't have anything on the request list but there
1014 * are buffers awaiting a flush, emit one and try again.
1015 * When we wait on it, those buffers waiting for that flush
1016 * will get moved to inactive.
1017 */
1018 if (!list_empty(&dev_priv->mm.flushing_list)) {
1019 obj_priv = list_first_entry(&dev_priv->mm.flushing_list,
1020 struct drm_i915_gem_object,
1021 list);
1022 obj = obj_priv->obj;
1023
1024 i915_gem_flush(dev,
1025 obj->write_domain,
1026 obj->write_domain);
1027 i915_add_request(dev, obj->write_domain);
1028
1029 obj = NULL;
1030 continue;
1031 }
1032
1033 DRM_ERROR("inactive empty %d request empty %d "
1034 "flushing empty %d\n",
1035 list_empty(&dev_priv->mm.inactive_list),
1036 list_empty(&dev_priv->mm.request_list),
1037 list_empty(&dev_priv->mm.flushing_list));
1038 /* If we didn't do any of the above, there's nothing to be done
1039 * and we just can't fit it in.
1040 */
1041 return -ENOMEM;
1042 }
1043 return ret;
1044}
1045
1046static int
1047i915_gem_object_get_page_list(struct drm_gem_object *obj)
1048{
1049 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1050 int page_count, i;
1051 struct address_space *mapping;
1052 struct inode *inode;
1053 struct page *page;
1054 int ret;
1055
1056 if (obj_priv->page_list)
1057 return 0;
1058
1059 /* Get the list of pages out of our struct file. They'll be pinned
1060 * at this point until we release them.
1061 */
1062 page_count = obj->size / PAGE_SIZE;
1063 BUG_ON(obj_priv->page_list != NULL);
1064 obj_priv->page_list = drm_calloc(page_count, sizeof(struct page *),
1065 DRM_MEM_DRIVER);
1066 if (obj_priv->page_list == NULL) {
1067 DRM_ERROR("Faled to allocate page list\n");
1068 return -ENOMEM;
1069 }
1070
1071 inode = obj->filp->f_path.dentry->d_inode;
1072 mapping = inode->i_mapping;
1073 for (i = 0; i < page_count; i++) {
1074 page = read_mapping_page(mapping, i, NULL);
1075 if (IS_ERR(page)) {
1076 ret = PTR_ERR(page);
1077 DRM_ERROR("read_mapping_page failed: %d\n", ret);
1078 i915_gem_object_free_page_list(obj);
1079 return ret;
1080 }
1081 obj_priv->page_list[i] = page;
1082 }
1083 return 0;
1084}
1085
1086/**
1087 * Finds free space in the GTT aperture and binds the object there.
1088 */
1089static int
1090i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, unsigned alignment)
1091{
1092 struct drm_device *dev = obj->dev;
1093 drm_i915_private_t *dev_priv = dev->dev_private;
1094 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1095 struct drm_mm_node *free_space;
1096 int page_count, ret;
1097
1098 if (alignment == 0)
1099 alignment = PAGE_SIZE;
1100 if (alignment & (PAGE_SIZE - 1)) {
1101 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1102 return -EINVAL;
1103 }
1104
1105 search_free:
1106 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1107 obj->size, alignment, 0);
1108 if (free_space != NULL) {
1109 obj_priv->gtt_space = drm_mm_get_block(free_space, obj->size,
1110 alignment);
1111 if (obj_priv->gtt_space != NULL) {
1112 obj_priv->gtt_space->private = obj;
1113 obj_priv->gtt_offset = obj_priv->gtt_space->start;
1114 }
1115 }
1116 if (obj_priv->gtt_space == NULL) {
1117 /* If the gtt is empty and we're still having trouble
1118 * fitting our object in, we're out of memory.
1119 */
1120#if WATCH_LRU
1121 DRM_INFO("%s: GTT full, evicting something\n", __func__);
1122#endif
1123 if (list_empty(&dev_priv->mm.inactive_list) &&
1124 list_empty(&dev_priv->mm.flushing_list) &&
1125 list_empty(&dev_priv->mm.active_list)) {
1126 DRM_ERROR("GTT full, but LRU list empty\n");
1127 return -ENOMEM;
1128 }
1129
1130 ret = i915_gem_evict_something(dev);
1131 if (ret != 0) {
1132 DRM_ERROR("Failed to evict a buffer %d\n", ret);
1133 return ret;
1134 }
1135 goto search_free;
1136 }
1137
1138#if WATCH_BUF
1139 DRM_INFO("Binding object of size %d at 0x%08x\n",
1140 obj->size, obj_priv->gtt_offset);
1141#endif
1142 ret = i915_gem_object_get_page_list(obj);
1143 if (ret) {
1144 drm_mm_put_block(obj_priv->gtt_space);
1145 obj_priv->gtt_space = NULL;
1146 return ret;
1147 }
1148
1149 page_count = obj->size / PAGE_SIZE;
1150 /* Create an AGP memory structure pointing at our pages, and bind it
1151 * into the GTT.
1152 */
1153 obj_priv->agp_mem = drm_agp_bind_pages(dev,
1154 obj_priv->page_list,
1155 page_count,
1156 obj_priv->gtt_offset);
1157 if (obj_priv->agp_mem == NULL) {
1158 i915_gem_object_free_page_list(obj);
1159 drm_mm_put_block(obj_priv->gtt_space);
1160 obj_priv->gtt_space = NULL;
1161 return -ENOMEM;
1162 }
1163 atomic_inc(&dev->gtt_count);
1164 atomic_add(obj->size, &dev->gtt_memory);
1165
1166 /* Assert that the object is not currently in any GPU domain. As it
1167 * wasn't in the GTT, there shouldn't be any way it could have been in
1168 * a GPU cache
1169 */
1170 BUG_ON(obj->read_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
1171 BUG_ON(obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
1172
1173 return 0;
1174}
1175
1176void
1177i915_gem_clflush_object(struct drm_gem_object *obj)
1178{
1179 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1180
1181 /* If we don't have a page list set up, then we're not pinned
1182 * to GPU, and we can ignore the cache flush because it'll happen
1183 * again at bind time.
1184 */
1185 if (obj_priv->page_list == NULL)
1186 return;
1187
1188 drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE);
1189}
1190
1191/*
1192 * Set the next domain for the specified object. This
1193 * may not actually perform the necessary flushing/invaliding though,
1194 * as that may want to be batched with other set_domain operations
1195 *
1196 * This is (we hope) the only really tricky part of gem. The goal
1197 * is fairly simple -- track which caches hold bits of the object
1198 * and make sure they remain coherent. A few concrete examples may
1199 * help to explain how it works. For shorthand, we use the notation
1200 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
1201 * a pair of read and write domain masks.
1202 *
1203 * Case 1: the batch buffer
1204 *
1205 * 1. Allocated
1206 * 2. Written by CPU
1207 * 3. Mapped to GTT
1208 * 4. Read by GPU
1209 * 5. Unmapped from GTT
1210 * 6. Freed
1211 *
1212 * Let's take these a step at a time
1213 *
1214 * 1. Allocated
1215 * Pages allocated from the kernel may still have
1216 * cache contents, so we set them to (CPU, CPU) always.
1217 * 2. Written by CPU (using pwrite)
1218 * The pwrite function calls set_domain (CPU, CPU) and
1219 * this function does nothing (as nothing changes)
1220 * 3. Mapped by GTT
1221 * This function asserts that the object is not
1222 * currently in any GPU-based read or write domains
1223 * 4. Read by GPU
1224 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
1225 * As write_domain is zero, this function adds in the
1226 * current read domains (CPU+COMMAND, 0).
1227 * flush_domains is set to CPU.
1228 * invalidate_domains is set to COMMAND
1229 * clflush is run to get data out of the CPU caches
1230 * then i915_dev_set_domain calls i915_gem_flush to
1231 * emit an MI_FLUSH and drm_agp_chipset_flush
1232 * 5. Unmapped from GTT
1233 * i915_gem_object_unbind calls set_domain (CPU, CPU)
1234 * flush_domains and invalidate_domains end up both zero
1235 * so no flushing/invalidating happens
1236 * 6. Freed
1237 * yay, done
1238 *
1239 * Case 2: The shared render buffer
1240 *
1241 * 1. Allocated
1242 * 2. Mapped to GTT
1243 * 3. Read/written by GPU
1244 * 4. set_domain to (CPU,CPU)
1245 * 5. Read/written by CPU
1246 * 6. Read/written by GPU
1247 *
1248 * 1. Allocated
1249 * Same as last example, (CPU, CPU)
1250 * 2. Mapped to GTT
1251 * Nothing changes (assertions find that it is not in the GPU)
1252 * 3. Read/written by GPU
1253 * execbuffer calls set_domain (RENDER, RENDER)
1254 * flush_domains gets CPU
1255 * invalidate_domains gets GPU
1256 * clflush (obj)
1257 * MI_FLUSH and drm_agp_chipset_flush
1258 * 4. set_domain (CPU, CPU)
1259 * flush_domains gets GPU
1260 * invalidate_domains gets CPU
1261 * wait_rendering (obj) to make sure all drawing is complete.
1262 * This will include an MI_FLUSH to get the data from GPU
1263 * to memory
1264 * clflush (obj) to invalidate the CPU cache
1265 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
1266 * 5. Read/written by CPU
1267 * cache lines are loaded and dirtied
1268 * 6. Read written by GPU
1269 * Same as last GPU access
1270 *
1271 * Case 3: The constant buffer
1272 *
1273 * 1. Allocated
1274 * 2. Written by CPU
1275 * 3. Read by GPU
1276 * 4. Updated (written) by CPU again
1277 * 5. Read by GPU
1278 *
1279 * 1. Allocated
1280 * (CPU, CPU)
1281 * 2. Written by CPU
1282 * (CPU, CPU)
1283 * 3. Read by GPU
1284 * (CPU+RENDER, 0)
1285 * flush_domains = CPU
1286 * invalidate_domains = RENDER
1287 * clflush (obj)
1288 * MI_FLUSH
1289 * drm_agp_chipset_flush
1290 * 4. Updated (written) by CPU again
1291 * (CPU, CPU)
1292 * flush_domains = 0 (no previous write domain)
1293 * invalidate_domains = 0 (no new read domains)
1294 * 5. Read by GPU
1295 * (CPU+RENDER, 0)
1296 * flush_domains = CPU
1297 * invalidate_domains = RENDER
1298 * clflush (obj)
1299 * MI_FLUSH
1300 * drm_agp_chipset_flush
1301 */
1302static int
1303i915_gem_object_set_domain(struct drm_gem_object *obj,
1304 uint32_t read_domains,
1305 uint32_t write_domain)
1306{
1307 struct drm_device *dev = obj->dev;
1308 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1309 uint32_t invalidate_domains = 0;
1310 uint32_t flush_domains = 0;
1311 int ret;
1312
1313#if WATCH_BUF
1314 DRM_INFO("%s: object %p read %08x -> %08x write %08x -> %08x\n",
1315 __func__, obj,
1316 obj->read_domains, read_domains,
1317 obj->write_domain, write_domain);
1318#endif
1319 /*
1320 * If the object isn't moving to a new write domain,
1321 * let the object stay in multiple read domains
1322 */
1323 if (write_domain == 0)
1324 read_domains |= obj->read_domains;
1325 else
1326 obj_priv->dirty = 1;
1327
1328 /*
1329 * Flush the current write domain if
1330 * the new read domains don't match. Invalidate
1331 * any read domains which differ from the old
1332 * write domain
1333 */
1334 if (obj->write_domain && obj->write_domain != read_domains) {
1335 flush_domains |= obj->write_domain;
1336 invalidate_domains |= read_domains & ~obj->write_domain;
1337 }
1338 /*
1339 * Invalidate any read caches which may have
1340 * stale data. That is, any new read domains.
1341 */
1342 invalidate_domains |= read_domains & ~obj->read_domains;
1343 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
1344#if WATCH_BUF
1345 DRM_INFO("%s: CPU domain flush %08x invalidate %08x\n",
1346 __func__, flush_domains, invalidate_domains);
1347#endif
1348 /*
1349 * If we're invaliding the CPU cache and flushing a GPU cache,
1350 * then pause for rendering so that the GPU caches will be
1351 * flushed before the cpu cache is invalidated
1352 */
1353 if ((invalidate_domains & I915_GEM_DOMAIN_CPU) &&
1354 (flush_domains & ~(I915_GEM_DOMAIN_CPU |
1355 I915_GEM_DOMAIN_GTT))) {
1356 ret = i915_gem_object_wait_rendering(obj);
1357 if (ret)
1358 return ret;
1359 }
1360 i915_gem_clflush_object(obj);
1361 }
1362
1363 if ((write_domain | flush_domains) != 0)
1364 obj->write_domain = write_domain;
1365
1366 /* If we're invalidating the CPU domain, clear the per-page CPU
1367 * domain list as well.
1368 */
1369 if (obj_priv->page_cpu_valid != NULL &&
1370 (write_domain != 0 ||
1371 read_domains & I915_GEM_DOMAIN_CPU)) {
1372 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
1373 DRM_MEM_DRIVER);
1374 obj_priv->page_cpu_valid = NULL;
1375 }
1376 obj->read_domains = read_domains;
1377
1378 dev->invalidate_domains |= invalidate_domains;
1379 dev->flush_domains |= flush_domains;
1380#if WATCH_BUF
1381 DRM_INFO("%s: read %08x write %08x invalidate %08x flush %08x\n",
1382 __func__,
1383 obj->read_domains, obj->write_domain,
1384 dev->invalidate_domains, dev->flush_domains);
1385#endif
1386 return 0;
1387}
1388
1389/**
1390 * Set the read/write domain on a range of the object.
1391 *
1392 * Currently only implemented for CPU reads, otherwise drops to normal
1393 * i915_gem_object_set_domain().
1394 */
1395static int
1396i915_gem_object_set_domain_range(struct drm_gem_object *obj,
1397 uint64_t offset,
1398 uint64_t size,
1399 uint32_t read_domains,
1400 uint32_t write_domain)
1401{
1402 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1403 int ret, i;
1404
1405 if (obj->read_domains & I915_GEM_DOMAIN_CPU)
1406 return 0;
1407
1408 if (read_domains != I915_GEM_DOMAIN_CPU ||
1409 write_domain != 0)
1410 return i915_gem_object_set_domain(obj,
1411 read_domains, write_domain);
1412
1413 /* Wait on any GPU rendering to the object to be flushed. */
1414 if (obj->write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT)) {
1415 ret = i915_gem_object_wait_rendering(obj);
1416 if (ret)
1417 return ret;
1418 }
1419
1420 if (obj_priv->page_cpu_valid == NULL) {
1421 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE,
1422 DRM_MEM_DRIVER);
1423 }
1424
1425 /* Flush the cache on any pages that are still invalid from the CPU's
1426 * perspective.
1427 */
1428 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; i++) {
1429 if (obj_priv->page_cpu_valid[i])
1430 continue;
1431
1432 drm_clflush_pages(obj_priv->page_list + i, 1);
1433
1434 obj_priv->page_cpu_valid[i] = 1;
1435 }
1436
1437 return 0;
1438}
1439
1440/**
1441 * Once all of the objects have been set in the proper domain,
1442 * perform the necessary flush and invalidate operations.
1443 *
1444 * Returns the write domains flushed, for use in flush tracking.
1445 */
1446static uint32_t
1447i915_gem_dev_set_domain(struct drm_device *dev)
1448{
1449 uint32_t flush_domains = dev->flush_domains;
1450
1451 /*
1452 * Now that all the buffers are synced to the proper domains,
1453 * flush and invalidate the collected domains
1454 */
1455 if (dev->invalidate_domains | dev->flush_domains) {
1456#if WATCH_EXEC
1457 DRM_INFO("%s: invalidate_domains %08x flush_domains %08x\n",
1458 __func__,
1459 dev->invalidate_domains,
1460 dev->flush_domains);
1461#endif
1462 i915_gem_flush(dev,
1463 dev->invalidate_domains,
1464 dev->flush_domains);
1465 dev->invalidate_domains = 0;
1466 dev->flush_domains = 0;
1467 }
1468
1469 return flush_domains;
1470}
1471
1472/**
1473 * Pin an object to the GTT and evaluate the relocations landing in it.
1474 */
1475static int
1476i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
1477 struct drm_file *file_priv,
1478 struct drm_i915_gem_exec_object *entry)
1479{
1480 struct drm_device *dev = obj->dev;
1481 struct drm_i915_gem_relocation_entry reloc;
1482 struct drm_i915_gem_relocation_entry __user *relocs;
1483 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1484 int i, ret;
1485 uint32_t last_reloc_offset = -1;
1486 void *reloc_page = NULL;
1487
1488 /* Choose the GTT offset for our buffer and put it there. */
1489 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
1490 if (ret)
1491 return ret;
1492
1493 entry->offset = obj_priv->gtt_offset;
1494
1495 relocs = (struct drm_i915_gem_relocation_entry __user *)
1496 (uintptr_t) entry->relocs_ptr;
1497 /* Apply the relocations, using the GTT aperture to avoid cache
1498 * flushing requirements.
1499 */
1500 for (i = 0; i < entry->relocation_count; i++) {
1501 struct drm_gem_object *target_obj;
1502 struct drm_i915_gem_object *target_obj_priv;
1503 uint32_t reloc_val, reloc_offset, *reloc_entry;
1504 int ret;
1505
1506 ret = copy_from_user(&reloc, relocs + i, sizeof(reloc));
1507 if (ret != 0) {
1508 i915_gem_object_unpin(obj);
1509 return ret;
1510 }
1511
1512 target_obj = drm_gem_object_lookup(obj->dev, file_priv,
1513 reloc.target_handle);
1514 if (target_obj == NULL) {
1515 i915_gem_object_unpin(obj);
1516 return -EBADF;
1517 }
1518 target_obj_priv = target_obj->driver_private;
1519
1520 /* The target buffer should have appeared before us in the
1521 * exec_object list, so it should have a GTT space bound by now.
1522 */
1523 if (target_obj_priv->gtt_space == NULL) {
1524 DRM_ERROR("No GTT space found for object %d\n",
1525 reloc.target_handle);
1526 drm_gem_object_unreference(target_obj);
1527 i915_gem_object_unpin(obj);
1528 return -EINVAL;
1529 }
1530
1531 if (reloc.offset > obj->size - 4) {
1532 DRM_ERROR("Relocation beyond object bounds: "
1533 "obj %p target %d offset %d size %d.\n",
1534 obj, reloc.target_handle,
1535 (int) reloc.offset, (int) obj->size);
1536 drm_gem_object_unreference(target_obj);
1537 i915_gem_object_unpin(obj);
1538 return -EINVAL;
1539 }
1540 if (reloc.offset & 3) {
1541 DRM_ERROR("Relocation not 4-byte aligned: "
1542 "obj %p target %d offset %d.\n",
1543 obj, reloc.target_handle,
1544 (int) reloc.offset);
1545 drm_gem_object_unreference(target_obj);
1546 i915_gem_object_unpin(obj);
1547 return -EINVAL;
1548 }
1549
1550 if (reloc.write_domain && target_obj->pending_write_domain &&
1551 reloc.write_domain != target_obj->pending_write_domain) {
1552 DRM_ERROR("Write domain conflict: "
1553 "obj %p target %d offset %d "
1554 "new %08x old %08x\n",
1555 obj, reloc.target_handle,
1556 (int) reloc.offset,
1557 reloc.write_domain,
1558 target_obj->pending_write_domain);
1559 drm_gem_object_unreference(target_obj);
1560 i915_gem_object_unpin(obj);
1561 return -EINVAL;
1562 }
1563
1564#if WATCH_RELOC
1565 DRM_INFO("%s: obj %p offset %08x target %d "
1566 "read %08x write %08x gtt %08x "
1567 "presumed %08x delta %08x\n",
1568 __func__,
1569 obj,
1570 (int) reloc.offset,
1571 (int) reloc.target_handle,
1572 (int) reloc.read_domains,
1573 (int) reloc.write_domain,
1574 (int) target_obj_priv->gtt_offset,
1575 (int) reloc.presumed_offset,
1576 reloc.delta);
1577#endif
1578
1579 target_obj->pending_read_domains |= reloc.read_domains;
1580 target_obj->pending_write_domain |= reloc.write_domain;
1581
1582 /* If the relocation already has the right value in it, no
1583 * more work needs to be done.
1584 */
1585 if (target_obj_priv->gtt_offset == reloc.presumed_offset) {
1586 drm_gem_object_unreference(target_obj);
1587 continue;
1588 }
1589
1590 /* Now that we're going to actually write some data in,
1591 * make sure that any rendering using this buffer's contents
1592 * is completed.
1593 */
1594 i915_gem_object_wait_rendering(obj);
1595
1596 /* As we're writing through the gtt, flush
1597 * any CPU writes before we write the relocations
1598 */
1599 if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
1600 i915_gem_clflush_object(obj);
1601 drm_agp_chipset_flush(dev);
1602 obj->write_domain = 0;
1603 }
1604
1605 /* Map the page containing the relocation we're going to
1606 * perform.
1607 */
1608 reloc_offset = obj_priv->gtt_offset + reloc.offset;
1609 if (reloc_page == NULL ||
1610 (last_reloc_offset & ~(PAGE_SIZE - 1)) !=
1611 (reloc_offset & ~(PAGE_SIZE - 1))) {
1612 if (reloc_page != NULL)
1613 iounmap(reloc_page);
1614
1615 reloc_page = ioremap(dev->agp->base +
1616 (reloc_offset & ~(PAGE_SIZE - 1)),
1617 PAGE_SIZE);
1618 last_reloc_offset = reloc_offset;
1619 if (reloc_page == NULL) {
1620 drm_gem_object_unreference(target_obj);
1621 i915_gem_object_unpin(obj);
1622 return -ENOMEM;
1623 }
1624 }
1625
1626 reloc_entry = (uint32_t *)((char *)reloc_page +
1627 (reloc_offset & (PAGE_SIZE - 1)));
1628 reloc_val = target_obj_priv->gtt_offset + reloc.delta;
1629
1630#if WATCH_BUF
1631 DRM_INFO("Applied relocation: %p@0x%08x %08x -> %08x\n",
1632 obj, (unsigned int) reloc.offset,
1633 readl(reloc_entry), reloc_val);
1634#endif
1635 writel(reloc_val, reloc_entry);
1636
1637 /* Write the updated presumed offset for this entry back out
1638 * to the user.
1639 */
1640 reloc.presumed_offset = target_obj_priv->gtt_offset;
1641 ret = copy_to_user(relocs + i, &reloc, sizeof(reloc));
1642 if (ret != 0) {
1643 drm_gem_object_unreference(target_obj);
1644 i915_gem_object_unpin(obj);
1645 return ret;
1646 }
1647
1648 drm_gem_object_unreference(target_obj);
1649 }
1650
1651 if (reloc_page != NULL)
1652 iounmap(reloc_page);
1653
1654#if WATCH_BUF
1655 if (0)
1656 i915_gem_dump_object(obj, 128, __func__, ~0);
1657#endif
1658 return 0;
1659}
1660
1661/** Dispatch a batchbuffer to the ring
1662 */
1663static int
1664i915_dispatch_gem_execbuffer(struct drm_device *dev,
1665 struct drm_i915_gem_execbuffer *exec,
1666 uint64_t exec_offset)
1667{
1668 drm_i915_private_t *dev_priv = dev->dev_private;
1669 struct drm_clip_rect __user *boxes = (struct drm_clip_rect __user *)
1670 (uintptr_t) exec->cliprects_ptr;
1671 int nbox = exec->num_cliprects;
1672 int i = 0, count;
1673 uint32_t exec_start, exec_len;
1674 RING_LOCALS;
1675
1676 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
1677 exec_len = (uint32_t) exec->batch_len;
1678
1679 if ((exec_start | exec_len) & 0x7) {
1680 DRM_ERROR("alignment\n");
1681 return -EINVAL;
1682 }
1683
1684 if (!exec_start)
1685 return -EINVAL;
1686
1687 count = nbox ? nbox : 1;
1688
1689 for (i = 0; i < count; i++) {
1690 if (i < nbox) {
1691 int ret = i915_emit_box(dev, boxes, i,
1692 exec->DR1, exec->DR4);
1693 if (ret)
1694 return ret;
1695 }
1696
1697 if (IS_I830(dev) || IS_845G(dev)) {
1698 BEGIN_LP_RING(4);
1699 OUT_RING(MI_BATCH_BUFFER);
1700 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1701 OUT_RING(exec_start + exec_len - 4);
1702 OUT_RING(0);
1703 ADVANCE_LP_RING();
1704 } else {
1705 BEGIN_LP_RING(2);
1706 if (IS_I965G(dev)) {
1707 OUT_RING(MI_BATCH_BUFFER_START |
1708 (2 << 6) |
1709 MI_BATCH_NON_SECURE_I965);
1710 OUT_RING(exec_start);
1711 } else {
1712 OUT_RING(MI_BATCH_BUFFER_START |
1713 (2 << 6));
1714 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1715 }
1716 ADVANCE_LP_RING();
1717 }
1718 }
1719
1720 /* XXX breadcrumb */
1721 return 0;
1722}
1723
1724/* Throttle our rendering by waiting until the ring has completed our requests
1725 * emitted over 20 msec ago.
1726 *
1727 * This should get us reasonable parallelism between CPU and GPU but also
1728 * relatively low latency when blocking on a particular request to finish.
1729 */
1730static int
1731i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
1732{
1733 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1734 int ret = 0;
1735 uint32_t seqno;
1736
1737 mutex_lock(&dev->struct_mutex);
1738 seqno = i915_file_priv->mm.last_gem_throttle_seqno;
1739 i915_file_priv->mm.last_gem_throttle_seqno =
1740 i915_file_priv->mm.last_gem_seqno;
1741 if (seqno)
1742 ret = i915_wait_request(dev, seqno);
1743 mutex_unlock(&dev->struct_mutex);
1744 return ret;
1745}
1746
1747int
1748i915_gem_execbuffer(struct drm_device *dev, void *data,
1749 struct drm_file *file_priv)
1750{
1751 drm_i915_private_t *dev_priv = dev->dev_private;
1752 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1753 struct drm_i915_gem_execbuffer *args = data;
1754 struct drm_i915_gem_exec_object *exec_list = NULL;
1755 struct drm_gem_object **object_list = NULL;
1756 struct drm_gem_object *batch_obj;
1757 int ret, i, pinned = 0;
1758 uint64_t exec_offset;
1759 uint32_t seqno, flush_domains;
1760
1761#if WATCH_EXEC
1762 DRM_INFO("buffers_ptr %d buffer_count %d len %08x\n",
1763 (int) args->buffers_ptr, args->buffer_count, args->batch_len);
1764#endif
1765
1766 /* Copy in the exec list from userland */
1767 exec_list = drm_calloc(sizeof(*exec_list), args->buffer_count,
1768 DRM_MEM_DRIVER);
1769 object_list = drm_calloc(sizeof(*object_list), args->buffer_count,
1770 DRM_MEM_DRIVER);
1771 if (exec_list == NULL || object_list == NULL) {
1772 DRM_ERROR("Failed to allocate exec or object list "
1773 "for %d buffers\n",
1774 args->buffer_count);
1775 ret = -ENOMEM;
1776 goto pre_mutex_err;
1777 }
1778 ret = copy_from_user(exec_list,
1779 (struct drm_i915_relocation_entry __user *)
1780 (uintptr_t) args->buffers_ptr,
1781 sizeof(*exec_list) * args->buffer_count);
1782 if (ret != 0) {
1783 DRM_ERROR("copy %d exec entries failed %d\n",
1784 args->buffer_count, ret);
1785 goto pre_mutex_err;
1786 }
1787
1788 mutex_lock(&dev->struct_mutex);
1789
1790 i915_verify_inactive(dev, __FILE__, __LINE__);
1791
1792 if (dev_priv->mm.wedged) {
1793 DRM_ERROR("Execbuf while wedged\n");
1794 mutex_unlock(&dev->struct_mutex);
1795 return -EIO;
1796 }
1797
1798 if (dev_priv->mm.suspended) {
1799 DRM_ERROR("Execbuf while VT-switched.\n");
1800 mutex_unlock(&dev->struct_mutex);
1801 return -EBUSY;
1802 }
1803
1804 /* Zero the gloabl flush/invalidate flags. These
1805 * will be modified as each object is bound to the
1806 * gtt
1807 */
1808 dev->invalidate_domains = 0;
1809 dev->flush_domains = 0;
1810
1811 /* Look up object handles and perform the relocations */
1812 for (i = 0; i < args->buffer_count; i++) {
1813 object_list[i] = drm_gem_object_lookup(dev, file_priv,
1814 exec_list[i].handle);
1815 if (object_list[i] == NULL) {
1816 DRM_ERROR("Invalid object handle %d at index %d\n",
1817 exec_list[i].handle, i);
1818 ret = -EBADF;
1819 goto err;
1820 }
1821
1822 object_list[i]->pending_read_domains = 0;
1823 object_list[i]->pending_write_domain = 0;
1824 ret = i915_gem_object_pin_and_relocate(object_list[i],
1825 file_priv,
1826 &exec_list[i]);
1827 if (ret) {
1828 DRM_ERROR("object bind and relocate failed %d\n", ret);
1829 goto err;
1830 }
1831 pinned = i + 1;
1832 }
1833
1834 /* Set the pending read domains for the batch buffer to COMMAND */
1835 batch_obj = object_list[args->buffer_count-1];
1836 batch_obj->pending_read_domains = I915_GEM_DOMAIN_COMMAND;
1837 batch_obj->pending_write_domain = 0;
1838
1839 i915_verify_inactive(dev, __FILE__, __LINE__);
1840
1841 for (i = 0; i < args->buffer_count; i++) {
1842 struct drm_gem_object *obj = object_list[i];
1843 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1844
1845 if (obj_priv->gtt_space == NULL) {
1846 /* We evicted the buffer in the process of validating
1847 * our set of buffers in. We could try to recover by
1848 * kicking them everything out and trying again from
1849 * the start.
1850 */
1851 ret = -ENOMEM;
1852 goto err;
1853 }
1854
1855 /* make sure all previous memory operations have passed */
1856 ret = i915_gem_object_set_domain(obj,
1857 obj->pending_read_domains,
1858 obj->pending_write_domain);
1859 if (ret)
1860 goto err;
1861 }
1862
1863 i915_verify_inactive(dev, __FILE__, __LINE__);
1864
1865 /* Flush/invalidate caches and chipset buffer */
1866 flush_domains = i915_gem_dev_set_domain(dev);
1867
1868 i915_verify_inactive(dev, __FILE__, __LINE__);
1869
1870#if WATCH_COHERENCY
1871 for (i = 0; i < args->buffer_count; i++) {
1872 i915_gem_object_check_coherency(object_list[i],
1873 exec_list[i].handle);
1874 }
1875#endif
1876
1877 exec_offset = exec_list[args->buffer_count - 1].offset;
1878
1879#if WATCH_EXEC
1880 i915_gem_dump_object(object_list[args->buffer_count - 1],
1881 args->batch_len,
1882 __func__,
1883 ~0);
1884#endif
1885
1886 (void)i915_add_request(dev, flush_domains);
1887
1888 /* Exec the batchbuffer */
1889 ret = i915_dispatch_gem_execbuffer(dev, args, exec_offset);
1890 if (ret) {
1891 DRM_ERROR("dispatch failed %d\n", ret);
1892 goto err;
1893 }
1894
1895 /*
1896 * Ensure that the commands in the batch buffer are
1897 * finished before the interrupt fires
1898 */
1899 flush_domains = i915_retire_commands(dev);
1900
1901 i915_verify_inactive(dev, __FILE__, __LINE__);
1902
1903 /*
1904 * Get a seqno representing the execution of the current buffer,
1905 * which we can wait on. We would like to mitigate these interrupts,
1906 * likely by only creating seqnos occasionally (so that we have
1907 * *some* interrupts representing completion of buffers that we can
1908 * wait on when trying to clear up gtt space).
1909 */
1910 seqno = i915_add_request(dev, flush_domains);
1911 BUG_ON(seqno == 0);
1912 i915_file_priv->mm.last_gem_seqno = seqno;
1913 for (i = 0; i < args->buffer_count; i++) {
1914 struct drm_gem_object *obj = object_list[i];
1915 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1916
1917 i915_gem_object_move_to_active(obj);
1918 obj_priv->last_rendering_seqno = seqno;
1919#if WATCH_LRU
1920 DRM_INFO("%s: move to exec list %p\n", __func__, obj);
1921#endif
1922 }
1923#if WATCH_LRU
1924 i915_dump_lru(dev, __func__);
1925#endif
1926
1927 i915_verify_inactive(dev, __FILE__, __LINE__);
1928
1929 /* Copy the new buffer offsets back to the user's exec list. */
1930 ret = copy_to_user((struct drm_i915_relocation_entry __user *)
1931 (uintptr_t) args->buffers_ptr,
1932 exec_list,
1933 sizeof(*exec_list) * args->buffer_count);
1934 if (ret)
1935 DRM_ERROR("failed to copy %d exec entries "
1936 "back to user (%d)\n",
1937 args->buffer_count, ret);
1938err:
1939 if (object_list != NULL) {
1940 for (i = 0; i < pinned; i++)
1941 i915_gem_object_unpin(object_list[i]);
1942
1943 for (i = 0; i < args->buffer_count; i++)
1944 drm_gem_object_unreference(object_list[i]);
1945 }
1946 mutex_unlock(&dev->struct_mutex);
1947
1948pre_mutex_err:
1949 drm_free(object_list, sizeof(*object_list) * args->buffer_count,
1950 DRM_MEM_DRIVER);
1951 drm_free(exec_list, sizeof(*exec_list) * args->buffer_count,
1952 DRM_MEM_DRIVER);
1953
1954 return ret;
1955}
1956
1957int
1958i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
1959{
1960 struct drm_device *dev = obj->dev;
1961 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1962 int ret;
1963
1964 i915_verify_inactive(dev, __FILE__, __LINE__);
1965 if (obj_priv->gtt_space == NULL) {
1966 ret = i915_gem_object_bind_to_gtt(obj, alignment);
1967 if (ret != 0) {
1968 DRM_ERROR("Failure to bind: %d", ret);
1969 return ret;
1970 }
1971 }
1972 obj_priv->pin_count++;
1973
1974 /* If the object is not active and not pending a flush,
1975 * remove it from the inactive list
1976 */
1977 if (obj_priv->pin_count == 1) {
1978 atomic_inc(&dev->pin_count);
1979 atomic_add(obj->size, &dev->pin_memory);
1980 if (!obj_priv->active &&
1981 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
1982 I915_GEM_DOMAIN_GTT)) == 0 &&
1983 !list_empty(&obj_priv->list))
1984 list_del_init(&obj_priv->list);
1985 }
1986 i915_verify_inactive(dev, __FILE__, __LINE__);
1987
1988 return 0;
1989}
1990
1991void
1992i915_gem_object_unpin(struct drm_gem_object *obj)
1993{
1994 struct drm_device *dev = obj->dev;
1995 drm_i915_private_t *dev_priv = dev->dev_private;
1996 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1997
1998 i915_verify_inactive(dev, __FILE__, __LINE__);
1999 obj_priv->pin_count--;
2000 BUG_ON(obj_priv->pin_count < 0);
2001 BUG_ON(obj_priv->gtt_space == NULL);
2002
2003 /* If the object is no longer pinned, and is
2004 * neither active nor being flushed, then stick it on
2005 * the inactive list
2006 */
2007 if (obj_priv->pin_count == 0) {
2008 if (!obj_priv->active &&
2009 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
2010 I915_GEM_DOMAIN_GTT)) == 0)
2011 list_move_tail(&obj_priv->list,
2012 &dev_priv->mm.inactive_list);
2013 atomic_dec(&dev->pin_count);
2014 atomic_sub(obj->size, &dev->pin_memory);
2015 }
2016 i915_verify_inactive(dev, __FILE__, __LINE__);
2017}
2018
2019int
2020i915_gem_pin_ioctl(struct drm_device *dev, void *data,
2021 struct drm_file *file_priv)
2022{
2023 struct drm_i915_gem_pin *args = data;
2024 struct drm_gem_object *obj;
2025 struct drm_i915_gem_object *obj_priv;
2026 int ret;
2027
2028 mutex_lock(&dev->struct_mutex);
2029
2030 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2031 if (obj == NULL) {
2032 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
2033 args->handle);
2034 mutex_unlock(&dev->struct_mutex);
2035 return -EBADF;
2036 }
2037 obj_priv = obj->driver_private;
2038
2039 ret = i915_gem_object_pin(obj, args->alignment);
2040 if (ret != 0) {
2041 drm_gem_object_unreference(obj);
2042 mutex_unlock(&dev->struct_mutex);
2043 return ret;
2044 }
2045
2046 /* XXX - flush the CPU caches for pinned objects
2047 * as the X server doesn't manage domains yet
2048 */
2049 if (obj->write_domain & I915_GEM_DOMAIN_CPU) {
2050 i915_gem_clflush_object(obj);
2051 drm_agp_chipset_flush(dev);
2052 obj->write_domain = 0;
2053 }
2054 args->offset = obj_priv->gtt_offset;
2055 drm_gem_object_unreference(obj);
2056 mutex_unlock(&dev->struct_mutex);
2057
2058 return 0;
2059}
2060
2061int
2062i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
2063 struct drm_file *file_priv)
2064{
2065 struct drm_i915_gem_pin *args = data;
2066 struct drm_gem_object *obj;
2067
2068 mutex_lock(&dev->struct_mutex);
2069
2070 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2071 if (obj == NULL) {
2072 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
2073 args->handle);
2074 mutex_unlock(&dev->struct_mutex);
2075 return -EBADF;
2076 }
2077
2078 i915_gem_object_unpin(obj);
2079
2080 drm_gem_object_unreference(obj);
2081 mutex_unlock(&dev->struct_mutex);
2082 return 0;
2083}
2084
2085int
2086i915_gem_busy_ioctl(struct drm_device *dev, void *data,
2087 struct drm_file *file_priv)
2088{
2089 struct drm_i915_gem_busy *args = data;
2090 struct drm_gem_object *obj;
2091 struct drm_i915_gem_object *obj_priv;
2092
2093 mutex_lock(&dev->struct_mutex);
2094 obj = drm_gem_object_lookup(dev, file_priv, args->handle);
2095 if (obj == NULL) {
2096 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
2097 args->handle);
2098 mutex_unlock(&dev->struct_mutex);
2099 return -EBADF;
2100 }
2101
2102 obj_priv = obj->driver_private;
2103 args->busy = obj_priv->active;
2104
2105 drm_gem_object_unreference(obj);
2106 mutex_unlock(&dev->struct_mutex);
2107 return 0;
2108}
2109
2110int
2111i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
2112 struct drm_file *file_priv)
2113{
2114 return i915_gem_ring_throttle(dev, file_priv);
2115}
2116
2117int i915_gem_init_object(struct drm_gem_object *obj)
2118{
2119 struct drm_i915_gem_object *obj_priv;
2120
2121 obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
2122 if (obj_priv == NULL)
2123 return -ENOMEM;
2124
2125 /*
2126 * We've just allocated pages from the kernel,
2127 * so they've just been written by the CPU with
2128 * zeros. They'll need to be clflushed before we
2129 * use them with the GPU.
2130 */
2131 obj->write_domain = I915_GEM_DOMAIN_CPU;
2132 obj->read_domains = I915_GEM_DOMAIN_CPU;
2133
2134 obj->driver_private = obj_priv;
2135 obj_priv->obj = obj;
2136 INIT_LIST_HEAD(&obj_priv->list);
2137 return 0;
2138}
2139
2140void i915_gem_free_object(struct drm_gem_object *obj)
2141{
2142 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2143
2144 while (obj_priv->pin_count > 0)
2145 i915_gem_object_unpin(obj);
2146
2147 i915_gem_object_unbind(obj);
2148
2149 drm_free(obj_priv->page_cpu_valid, 1, DRM_MEM_DRIVER);
2150 drm_free(obj->driver_private, 1, DRM_MEM_DRIVER);
2151}
2152
2153static int
2154i915_gem_set_domain(struct drm_gem_object *obj,
2155 struct drm_file *file_priv,
2156 uint32_t read_domains,
2157 uint32_t write_domain)
2158{
2159 struct drm_device *dev = obj->dev;
2160 int ret;
2161 uint32_t flush_domains;
2162
2163 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
2164
2165 ret = i915_gem_object_set_domain(obj, read_domains, write_domain);
2166 if (ret)
2167 return ret;
2168 flush_domains = i915_gem_dev_set_domain(obj->dev);
2169
2170 if (flush_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT))
2171 (void) i915_add_request(dev, flush_domains);
2172
2173 return 0;
2174}
2175
2176/** Unbinds all objects that are on the given buffer list. */
2177static int
2178i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head)
2179{
2180 struct drm_gem_object *obj;
2181 struct drm_i915_gem_object *obj_priv;
2182 int ret;
2183
2184 while (!list_empty(head)) {
2185 obj_priv = list_first_entry(head,
2186 struct drm_i915_gem_object,
2187 list);
2188 obj = obj_priv->obj;
2189
2190 if (obj_priv->pin_count != 0) {
2191 DRM_ERROR("Pinned object in unbind list\n");
2192 mutex_unlock(&dev->struct_mutex);
2193 return -EINVAL;
2194 }
2195
2196 ret = i915_gem_object_unbind(obj);
2197 if (ret != 0) {
2198 DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
2199 ret);
2200 mutex_unlock(&dev->struct_mutex);
2201 return ret;
2202 }
2203 }
2204
2205
2206 return 0;
2207}
2208
2209static int
2210i915_gem_idle(struct drm_device *dev)
2211{
2212 drm_i915_private_t *dev_priv = dev->dev_private;
2213 uint32_t seqno, cur_seqno, last_seqno;
2214 int stuck, ret;
2215
2216 if (dev_priv->mm.suspended)
2217 return 0;
2218
2219 /* Hack! Don't let anybody do execbuf while we don't control the chip.
2220 * We need to replace this with a semaphore, or something.
2221 */
2222 dev_priv->mm.suspended = 1;
2223
2224 i915_kernel_lost_context(dev);
2225
2226 /* Flush the GPU along with all non-CPU write domains
2227 */
2228 i915_gem_flush(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT),
2229 ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
2230 seqno = i915_add_request(dev, ~(I915_GEM_DOMAIN_CPU |
2231 I915_GEM_DOMAIN_GTT));
2232
2233 if (seqno == 0) {
2234 mutex_unlock(&dev->struct_mutex);
2235 return -ENOMEM;
2236 }
2237
2238 dev_priv->mm.waiting_gem_seqno = seqno;
2239 last_seqno = 0;
2240 stuck = 0;
2241 for (;;) {
2242 cur_seqno = i915_get_gem_seqno(dev);
2243 if (i915_seqno_passed(cur_seqno, seqno))
2244 break;
2245 if (last_seqno == cur_seqno) {
2246 if (stuck++ > 100) {
2247 DRM_ERROR("hardware wedged\n");
2248 dev_priv->mm.wedged = 1;
2249 DRM_WAKEUP(&dev_priv->irq_queue);
2250 break;
2251 }
2252 }
2253 msleep(10);
2254 last_seqno = cur_seqno;
2255 }
2256 dev_priv->mm.waiting_gem_seqno = 0;
2257
2258 i915_gem_retire_requests(dev);
2259
2260 /* Active and flushing should now be empty as we've
2261 * waited for a sequence higher than any pending execbuffer
2262 */
2263 BUG_ON(!list_empty(&dev_priv->mm.active_list));
2264 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2265
2266 /* Request should now be empty as we've also waited
2267 * for the last request in the list
2268 */
2269 BUG_ON(!list_empty(&dev_priv->mm.request_list));
2270
2271 /* Move all buffers out of the GTT. */
2272 ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list);
2273 if (ret)
2274 return ret;
2275
2276 BUG_ON(!list_empty(&dev_priv->mm.active_list));
2277 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2278 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
2279 BUG_ON(!list_empty(&dev_priv->mm.request_list));
2280 return 0;
2281}
2282
2283static int
2284i915_gem_init_hws(struct drm_device *dev)
2285{
2286 drm_i915_private_t *dev_priv = dev->dev_private;
2287 struct drm_gem_object *obj;
2288 struct drm_i915_gem_object *obj_priv;
2289 int ret;
2290
2291 /* If we need a physical address for the status page, it's already
2292 * initialized at driver load time.
2293 */
2294 if (!I915_NEED_GFX_HWS(dev))
2295 return 0;
2296
2297 obj = drm_gem_object_alloc(dev, 4096);
2298 if (obj == NULL) {
2299 DRM_ERROR("Failed to allocate status page\n");
2300 return -ENOMEM;
2301 }
2302 obj_priv = obj->driver_private;
2303
2304 ret = i915_gem_object_pin(obj, 4096);
2305 if (ret != 0) {
2306 drm_gem_object_unreference(obj);
2307 return ret;
2308 }
2309
2310 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
2311 dev_priv->hws_map.offset = dev->agp->base + obj_priv->gtt_offset;
2312 dev_priv->hws_map.size = 4096;
2313 dev_priv->hws_map.type = 0;
2314 dev_priv->hws_map.flags = 0;
2315 dev_priv->hws_map.mtrr = 0;
2316
2317 drm_core_ioremap(&dev_priv->hws_map, dev);
2318 if (dev_priv->hws_map.handle == NULL) {
2319 DRM_ERROR("Failed to map status page.\n");
2320 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2321 drm_gem_object_unreference(obj);
2322 return -EINVAL;
2323 }
2324 dev_priv->hws_obj = obj;
2325 dev_priv->hw_status_page = dev_priv->hws_map.handle;
2326 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
2327 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
2328 DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
2329
2330 return 0;
2331}
2332
2333static int
2334i915_gem_init_ringbuffer(struct drm_device *dev)
2335{
2336 drm_i915_private_t *dev_priv = dev->dev_private;
2337 struct drm_gem_object *obj;
2338 struct drm_i915_gem_object *obj_priv;
2339 int ret;
2340
2341 ret = i915_gem_init_hws(dev);
2342 if (ret != 0)
2343 return ret;
2344
2345 obj = drm_gem_object_alloc(dev, 128 * 1024);
2346 if (obj == NULL) {
2347 DRM_ERROR("Failed to allocate ringbuffer\n");
2348 return -ENOMEM;
2349 }
2350 obj_priv = obj->driver_private;
2351
2352 ret = i915_gem_object_pin(obj, 4096);
2353 if (ret != 0) {
2354 drm_gem_object_unreference(obj);
2355 return ret;
2356 }
2357
2358 /* Set up the kernel mapping for the ring. */
2359 dev_priv->ring.Size = obj->size;
2360 dev_priv->ring.tail_mask = obj->size - 1;
2361
2362 dev_priv->ring.map.offset = dev->agp->base + obj_priv->gtt_offset;
2363 dev_priv->ring.map.size = obj->size;
2364 dev_priv->ring.map.type = 0;
2365 dev_priv->ring.map.flags = 0;
2366 dev_priv->ring.map.mtrr = 0;
2367
2368 drm_core_ioremap(&dev_priv->ring.map, dev);
2369 if (dev_priv->ring.map.handle == NULL) {
2370 DRM_ERROR("Failed to map ringbuffer.\n");
2371 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2372 drm_gem_object_unreference(obj);
2373 return -EINVAL;
2374 }
2375 dev_priv->ring.ring_obj = obj;
2376 dev_priv->ring.virtual_start = dev_priv->ring.map.handle;
2377
2378 /* Stop the ring if it's running. */
2379 I915_WRITE(PRB0_CTL, 0);
2380 I915_WRITE(PRB0_HEAD, 0);
2381 I915_WRITE(PRB0_TAIL, 0);
2382 I915_WRITE(PRB0_START, 0);
2383
2384 /* Initialize the ring. */
2385 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
2386 I915_WRITE(PRB0_CTL,
2387 ((obj->size - 4096) & RING_NR_PAGES) |
2388 RING_NO_REPORT |
2389 RING_VALID);
2390
2391 /* Update our cache of the ring state */
2392 i915_kernel_lost_context(dev);
2393
2394 return 0;
2395}
2396
2397static void
2398i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2399{
2400 drm_i915_private_t *dev_priv = dev->dev_private;
2401
2402 if (dev_priv->ring.ring_obj == NULL)
2403 return;
2404
2405 drm_core_ioremapfree(&dev_priv->ring.map, dev);
2406
2407 i915_gem_object_unpin(dev_priv->ring.ring_obj);
2408 drm_gem_object_unreference(dev_priv->ring.ring_obj);
2409 dev_priv->ring.ring_obj = NULL;
2410 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2411
2412 if (dev_priv->hws_obj != NULL) {
2413 i915_gem_object_unpin(dev_priv->hws_obj);
2414 drm_gem_object_unreference(dev_priv->hws_obj);
2415 dev_priv->hws_obj = NULL;
2416 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2417
2418 /* Write high address into HWS_PGA when disabling. */
2419 I915_WRITE(HWS_PGA, 0x1ffff000);
2420 }
2421}
2422
2423int
2424i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
2425 struct drm_file *file_priv)
2426{
2427 drm_i915_private_t *dev_priv = dev->dev_private;
2428 int ret;
2429
2430 if (dev_priv->mm.wedged) {
2431 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2432 dev_priv->mm.wedged = 0;
2433 }
2434
2435 ret = i915_gem_init_ringbuffer(dev);
2436 if (ret != 0)
2437 return ret;
2438
2439 mutex_lock(&dev->struct_mutex);
2440 BUG_ON(!list_empty(&dev_priv->mm.active_list));
2441 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
2442 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
2443 BUG_ON(!list_empty(&dev_priv->mm.request_list));
2444 dev_priv->mm.suspended = 0;
2445 mutex_unlock(&dev->struct_mutex);
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04002446
2447 drm_irq_install(dev);
2448
Eric Anholt673a3942008-07-30 12:06:12 -07002449 return 0;
2450}
2451
2452int
2453i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
2454 struct drm_file *file_priv)
2455{
2456 int ret;
2457
2458 mutex_lock(&dev->struct_mutex);
2459 ret = i915_gem_idle(dev);
2460 if (ret == 0)
2461 i915_gem_cleanup_ringbuffer(dev);
2462 mutex_unlock(&dev->struct_mutex);
2463
Kristian Høgsbergdbb19d32008-08-20 11:04:27 -04002464 drm_irq_uninstall(dev);
2465
Eric Anholt673a3942008-07-30 12:06:12 -07002466 return 0;
2467}
2468
2469void
2470i915_gem_lastclose(struct drm_device *dev)
2471{
2472 int ret;
2473 drm_i915_private_t *dev_priv = dev->dev_private;
2474
2475 mutex_lock(&dev->struct_mutex);
2476
2477 if (dev_priv->ring.ring_obj != NULL) {
2478 ret = i915_gem_idle(dev);
2479 if (ret)
2480 DRM_ERROR("failed to idle hardware: %d\n", ret);
2481
2482 i915_gem_cleanup_ringbuffer(dev);
2483 }
2484
2485 mutex_unlock(&dev->struct_mutex);
2486}
2487
2488void
2489i915_gem_load(struct drm_device *dev)
2490{
2491 drm_i915_private_t *dev_priv = dev->dev_private;
2492
2493 INIT_LIST_HEAD(&dev_priv->mm.active_list);
2494 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
2495 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
2496 INIT_LIST_HEAD(&dev_priv->mm.request_list);
2497 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
2498 i915_gem_retire_work_handler);
Eric Anholt546b0972008-09-01 16:45:29 -07002499 INIT_WORK(&dev_priv->mm.vblank_work,
2500 i915_gem_vblank_work_handler);
Eric Anholt673a3942008-07-30 12:06:12 -07002501 dev_priv->mm.next_gem_seqno = 1;
2502
2503 i915_gem_detect_bit_6_swizzle(dev);
2504}