blob: 13a796fafaee2d71304d57e4e74574241f04dc0e [file] [log] [blame]
Eric Anholt62fdfea2010-05-21 13:26:39 -07001/*
2 * Copyright © 2008-2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Zou Nan hai <nanhai.zou@intel.com>
26 * Xiang Hai hao<haihao.xiang@intel.com>
27 *
28 */
29
30#include "drmP.h"
31#include "drm.h"
32#include "i915_drm.h"
33#include "i915_drv.h"
34#include "i915_trace.h"
35#include "intel_drv.h"
36
37void
38i915_gem_flush(struct drm_device *dev,
39 uint32_t invalidate_domains,
40 uint32_t flush_domains)
41{
42 drm_i915_private_t *dev_priv = dev->dev_private;
43 uint32_t cmd;
44 RING_LOCALS;
45
46#if WATCH_EXEC
47 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
48 invalidate_domains, flush_domains);
49#endif
50 trace_i915_gem_request_flush(dev, dev_priv->mm.next_gem_seqno,
51 invalidate_domains, flush_domains);
52
53 if (flush_domains & I915_GEM_DOMAIN_CPU)
54 drm_agp_chipset_flush(dev);
55
56 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
57 /*
58 * read/write caches:
59 *
60 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
61 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
62 * also flushed at 2d versus 3d pipeline switches.
63 *
64 * read-only caches:
65 *
66 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
67 * MI_READ_FLUSH is set, and is always flushed on 965.
68 *
69 * I915_GEM_DOMAIN_COMMAND may not exist?
70 *
71 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
72 * invalidated when MI_EXE_FLUSH is set.
73 *
74 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
75 * invalidated with every MI_FLUSH.
76 *
77 * TLBs:
78 *
79 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
80 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
81 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
82 * are flushed at any MI_FLUSH.
83 */
84
85 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
86 if ((invalidate_domains|flush_domains) &
87 I915_GEM_DOMAIN_RENDER)
88 cmd &= ~MI_NO_WRITE_FLUSH;
89 if (!IS_I965G(dev)) {
90 /*
91 * On the 965, the sampler cache always gets flushed
92 * and this bit is reserved.
93 */
94 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
95 cmd |= MI_READ_FLUSH;
96 }
97 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
98 cmd |= MI_EXE_FLUSH;
99
100#if WATCH_EXEC
101 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
102#endif
103 BEGIN_LP_RING(2);
104 OUT_RING(cmd);
105 OUT_RING(MI_NOOP);
106 ADVANCE_LP_RING();
107 }
108
109}
110#define PIPE_CONTROL_FLUSH(addr) \
111 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
112 PIPE_CONTROL_DEPTH_STALL); \
113 OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT); \
114 OUT_RING(0); \
115 OUT_RING(0); \
116
117/**
118 * Creates a new sequence number, emitting a write of it to the status page
119 * plus an interrupt, which will trigger i915_user_interrupt_handler.
120 *
121 * Must be called with struct_lock held.
122 *
123 * Returned sequence numbers are nonzero on success.
124 */
125uint32_t
126i915_ring_add_request(struct drm_device *dev)
127{
128 drm_i915_private_t *dev_priv = dev->dev_private;
129 uint32_t seqno;
130 RING_LOCALS;
131
132 /* Grab the seqno we're going to make this request be, and bump the
133 * next (skipping 0 so it can be the reserved no-seqno value).
134 */
135 seqno = dev_priv->mm.next_gem_seqno;
136 dev_priv->mm.next_gem_seqno++;
137 if (dev_priv->mm.next_gem_seqno == 0)
138 dev_priv->mm.next_gem_seqno++;
139
140 if (HAS_PIPE_CONTROL(dev)) {
141 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
142
143 /*
144 * Workaround qword write incoherence by flushing the
145 * PIPE_NOTIFY buffers out to memory before requesting
146 * an interrupt.
147 */
148 BEGIN_LP_RING(32);
149 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
150 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
151 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
152 OUT_RING(seqno);
153 OUT_RING(0);
154 PIPE_CONTROL_FLUSH(scratch_addr);
155 scratch_addr += 128; /* write to separate cachelines */
156 PIPE_CONTROL_FLUSH(scratch_addr);
157 scratch_addr += 128;
158 PIPE_CONTROL_FLUSH(scratch_addr);
159 scratch_addr += 128;
160 PIPE_CONTROL_FLUSH(scratch_addr);
161 scratch_addr += 128;
162 PIPE_CONTROL_FLUSH(scratch_addr);
163 scratch_addr += 128;
164 PIPE_CONTROL_FLUSH(scratch_addr);
165 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
166 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
167 PIPE_CONTROL_NOTIFY);
168 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
169 OUT_RING(seqno);
170 OUT_RING(0);
171 ADVANCE_LP_RING();
172 } else {
173 BEGIN_LP_RING(4);
174 OUT_RING(MI_STORE_DWORD_INDEX);
175 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
176 OUT_RING(seqno);
177
178 OUT_RING(MI_USER_INTERRUPT);
179 ADVANCE_LP_RING();
180 }
181 return seqno;
182}
183
184void i915_user_irq_get(struct drm_device *dev)
185{
186 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
187 unsigned long irqflags;
188
189 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
190 if (dev->irq_enabled && (++dev_priv->user_irq_refcount == 1)) {
191 if (HAS_PCH_SPLIT(dev))
192 ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
193 else
194 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
195 }
196 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
197}
198
199void i915_user_irq_put(struct drm_device *dev)
200{
201 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
202 unsigned long irqflags;
203
204 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
205 BUG_ON(dev->irq_enabled && dev_priv->user_irq_refcount <= 0);
206 if (dev->irq_enabled && (--dev_priv->user_irq_refcount == 0)) {
207 if (HAS_PCH_SPLIT(dev))
208 ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
209 else
210 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
211 }
212 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
213}
214
215/** Dispatch a batchbuffer to the ring
216 */
217int
218i915_dispatch_gem_execbuffer(struct drm_device *dev,
219 struct drm_i915_gem_execbuffer2 *exec,
220 struct drm_clip_rect *cliprects,
221 uint64_t exec_offset)
222{
223 drm_i915_private_t *dev_priv = dev->dev_private;
224 int nbox = exec->num_cliprects;
225 int i = 0, count;
226 uint32_t exec_start, exec_len;
227 RING_LOCALS;
228
229 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
230 exec_len = (uint32_t) exec->batch_len;
231
232 trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
233
234 count = nbox ? nbox : 1;
235
236 for (i = 0; i < count; i++) {
237 if (i < nbox) {
238 int ret = i915_emit_box(dev, cliprects, i,
239 exec->DR1, exec->DR4);
240 if (ret)
241 return ret;
242 }
243
244 if (IS_I830(dev) || IS_845G(dev)) {
245 BEGIN_LP_RING(4);
246 OUT_RING(MI_BATCH_BUFFER);
247 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
248 OUT_RING(exec_start + exec_len - 4);
249 OUT_RING(0);
250 ADVANCE_LP_RING();
251 } else {
252 BEGIN_LP_RING(2);
253 if (IS_I965G(dev)) {
254 OUT_RING(MI_BATCH_BUFFER_START |
255 (2 << 6) |
256 MI_BATCH_NON_SECURE_I965);
257 OUT_RING(exec_start);
258 } else {
259 OUT_RING(MI_BATCH_BUFFER_START |
260 (2 << 6));
261 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
262 }
263 ADVANCE_LP_RING();
264 }
265 }
266
267 /* XXX breadcrumb */
268 return 0;
269}
270
271static void
272i915_gem_cleanup_hws(struct drm_device *dev)
273{
274 drm_i915_private_t *dev_priv = dev->dev_private;
275 struct drm_gem_object *obj;
276 struct drm_i915_gem_object *obj_priv;
277
278 if (dev_priv->hws_obj == NULL)
279 return;
280
281 obj = dev_priv->hws_obj;
282 obj_priv = to_intel_bo(obj);
283
284 kunmap(obj_priv->pages[0]);
285 i915_gem_object_unpin(obj);
286 drm_gem_object_unreference(obj);
287 dev_priv->hws_obj = NULL;
288
289 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
290 dev_priv->hw_status_page = NULL;
291
292 if (HAS_PIPE_CONTROL(dev))
293 i915_gem_cleanup_pipe_control(dev);
294
295 /* Write high address into HWS_PGA when disabling. */
296 I915_WRITE(HWS_PGA, 0x1ffff000);
297}
298
299static int
300i915_gem_init_hws(struct drm_device *dev)
301{
302 drm_i915_private_t *dev_priv = dev->dev_private;
303 struct drm_gem_object *obj;
304 struct drm_i915_gem_object *obj_priv;
305 int ret;
306
307 /* If we need a physical address for the status page, it's already
308 * initialized at driver load time.
309 */
310 if (!I915_NEED_GFX_HWS(dev))
311 return 0;
312
313 obj = i915_gem_alloc_object(dev, 4096);
314 if (obj == NULL) {
315 DRM_ERROR("Failed to allocate status page\n");
316 ret = -ENOMEM;
317 goto err;
318 }
319 obj_priv = to_intel_bo(obj);
320 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
321
322 ret = i915_gem_object_pin(obj, 4096);
323 if (ret != 0) {
324 drm_gem_object_unreference(obj);
325 goto err_unref;
326 }
327
328 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
329
330 dev_priv->hw_status_page = kmap(obj_priv->pages[0]);
331 if (dev_priv->hw_status_page == NULL) {
332 DRM_ERROR("Failed to map status page.\n");
333 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
334 ret = -EINVAL;
335 goto err_unpin;
336 }
337
338 if (HAS_PIPE_CONTROL(dev)) {
339 ret = i915_gem_init_pipe_control(dev);
340 if (ret)
341 goto err_unpin;
342 }
343
344 dev_priv->hws_obj = obj;
345 memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
346 if (IS_GEN6(dev)) {
347 I915_WRITE(HWS_PGA_GEN6, dev_priv->status_gfx_addr);
348 I915_READ(HWS_PGA_GEN6); /* posting read */
349 } else {
350 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
351 I915_READ(HWS_PGA); /* posting read */
352 }
353 DRM_DEBUG_DRIVER("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
354
355 return 0;
356
357err_unpin:
358 i915_gem_object_unpin(obj);
359err_unref:
360 drm_gem_object_unreference(obj);
361err:
362 return 0;
363}
364
365int
366i915_gem_init_ringbuffer(struct drm_device *dev)
367{
368 drm_i915_private_t *dev_priv = dev->dev_private;
369 struct drm_gem_object *obj;
370 struct drm_i915_gem_object *obj_priv;
371 drm_i915_ring_buffer_t *ring = &dev_priv->ring;
372 int ret;
373 u32 head;
374
375 ret = i915_gem_init_hws(dev);
376 if (ret != 0)
377 return ret;
378
379 obj = i915_gem_alloc_object(dev, 128 * 1024);
380 if (obj == NULL) {
381 DRM_ERROR("Failed to allocate ringbuffer\n");
382 i915_gem_cleanup_hws(dev);
383 return -ENOMEM;
384 }
385 obj_priv = to_intel_bo(obj);
386
387 ret = i915_gem_object_pin(obj, 4096);
388 if (ret != 0) {
389 drm_gem_object_unreference(obj);
390 i915_gem_cleanup_hws(dev);
391 return ret;
392 }
393
394 /* Set up the kernel mapping for the ring. */
395 ring->Size = obj->size;
396
397 ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
398 ring->map.size = obj->size;
399 ring->map.type = 0;
400 ring->map.flags = 0;
401 ring->map.mtrr = 0;
402
403 drm_core_ioremap_wc(&ring->map, dev);
404 if (ring->map.handle == NULL) {
405 DRM_ERROR("Failed to map ringbuffer.\n");
406 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
407 i915_gem_object_unpin(obj);
408 drm_gem_object_unreference(obj);
409 i915_gem_cleanup_hws(dev);
410 return -EINVAL;
411 }
412 ring->ring_obj = obj;
413 ring->virtual_start = ring->map.handle;
414
415 /* Stop the ring if it's running. */
416 I915_WRITE(PRB0_CTL, 0);
417 I915_WRITE(PRB0_TAIL, 0);
418 I915_WRITE(PRB0_HEAD, 0);
419
420 /* Initialize the ring. */
421 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
422 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
423
424 /* G45 ring initialization fails to reset head to zero */
425 if (head != 0) {
426 DRM_ERROR("Ring head not reset to zero "
427 "ctl %08x head %08x tail %08x start %08x\n",
428 I915_READ(PRB0_CTL),
429 I915_READ(PRB0_HEAD),
430 I915_READ(PRB0_TAIL),
431 I915_READ(PRB0_START));
432 I915_WRITE(PRB0_HEAD, 0);
433
434 DRM_ERROR("Ring head forced to zero "
435 "ctl %08x head %08x tail %08x start %08x\n",
436 I915_READ(PRB0_CTL),
437 I915_READ(PRB0_HEAD),
438 I915_READ(PRB0_TAIL),
439 I915_READ(PRB0_START));
440 }
441
442 I915_WRITE(PRB0_CTL,
443 ((obj->size - 4096) & RING_NR_PAGES) |
444 RING_NO_REPORT |
445 RING_VALID);
446
447 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
448
449 /* If the head is still not zero, the ring is dead */
450 if (head != 0) {
451 DRM_ERROR("Ring initialization failed "
452 "ctl %08x head %08x tail %08x start %08x\n",
453 I915_READ(PRB0_CTL),
454 I915_READ(PRB0_HEAD),
455 I915_READ(PRB0_TAIL),
456 I915_READ(PRB0_START));
457 return -EIO;
458 }
459
460 /* Update our cache of the ring state */
461 if (!drm_core_check_feature(dev, DRIVER_MODESET))
462 i915_kernel_lost_context(dev);
463 else {
464 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
465 ring->tail = I915_READ(PRB0_TAIL) & TAIL_ADDR;
466 ring->space = ring->head - (ring->tail + 8);
467 if (ring->space < 0)
468 ring->space += ring->Size;
469 }
470
471 if (IS_I9XX(dev) && !IS_GEN3(dev)) {
472 I915_WRITE(MI_MODE,
473 (VS_TIMER_DISPATCH) << 16 | VS_TIMER_DISPATCH);
474 }
475
476 return 0;
477}
478
479void
480i915_gem_cleanup_ringbuffer(struct drm_device *dev)
481{
482 drm_i915_private_t *dev_priv = dev->dev_private;
483
484 if (dev_priv->ring.ring_obj == NULL)
485 return;
486
487 drm_core_ioremapfree(&dev_priv->ring.map, dev);
488
489 i915_gem_object_unpin(dev_priv->ring.ring_obj);
490 drm_gem_object_unreference(dev_priv->ring.ring_obj);
491 dev_priv->ring.ring_obj = NULL;
492 memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
493
494 i915_gem_cleanup_hws(dev);
495}
496
497/* As a ringbuffer is only allowed to wrap between instructions, fill
498 * the tail with NOOPs.
499 */
500int i915_wrap_ring(struct drm_device *dev)
501{
502 drm_i915_private_t *dev_priv = dev->dev_private;
503 volatile unsigned int *virt;
504 int rem;
505
506 rem = dev_priv->ring.Size - dev_priv->ring.tail;
507 if (dev_priv->ring.space < rem) {
508 int ret = i915_wait_ring(dev, rem, __func__);
509 if (ret)
510 return ret;
511 }
512 dev_priv->ring.space -= rem;
513
514 virt = (unsigned int *)
515 (dev_priv->ring.virtual_start + dev_priv->ring.tail);
516 rem /= 4;
517 while (rem--)
518 *virt++ = MI_NOOP;
519
520 dev_priv->ring.tail = 0;
521
522 return 0;
523}
524
525int i915_wait_ring(struct drm_device * dev, int n, const char *caller)
526{
527 drm_i915_private_t *dev_priv = dev->dev_private;
528 drm_i915_ring_buffer_t *ring = &(dev_priv->ring);
529 u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
530 u32 last_acthd = I915_READ(acthd_reg);
531 u32 acthd;
532 u32 last_head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
533 int i;
534
535 trace_i915_ring_wait_begin (dev);
536
537 for (i = 0; i < 100000; i++) {
538 ring->head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
539 acthd = I915_READ(acthd_reg);
540 ring->space = ring->head - (ring->tail + 8);
541 if (ring->space < 0)
542 ring->space += ring->Size;
543 if (ring->space >= n) {
544 trace_i915_ring_wait_end (dev);
545 return 0;
546 }
547
548 if (dev->primary->master) {
549 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
550 if (master_priv->sarea_priv)
551 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
552 }
553
554
555 if (ring->head != last_head)
556 i = 0;
557 if (acthd != last_acthd)
558 i = 0;
559
560 last_head = ring->head;
561 last_acthd = acthd;
562 msleep_interruptible(10);
563
564 }
565
566 trace_i915_ring_wait_end (dev);
567 return -EBUSY;
568}