blob: 7ab72af045ec70ca4baaf09b1eb9032836b12a1a [file] [log] [blame]
Eric Anholt62fdfea2010-05-21 13:26:39 -07001/*
2 * Copyright © 2008-2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Zou Nan hai <nanhai.zou@intel.com>
26 * Xiang Hai hao<haihao.xiang@intel.com>
27 *
28 */
29
30#include "drmP.h"
31#include "drm.h"
Eric Anholt62fdfea2010-05-21 13:26:39 -070032#include "i915_drv.h"
Zou Nan hai8187a2b2010-05-21 09:08:55 +080033#include "i915_drm.h"
Eric Anholt62fdfea2010-05-21 13:26:39 -070034#include "i915_trace.h"
Eric Anholt62fdfea2010-05-21 13:26:39 -070035
Zou Nan hai8187a2b2010-05-21 09:08:55 +080036static void
37render_ring_flush(struct drm_device *dev,
38 struct intel_ring_buffer *ring,
39 u32 invalidate_domains,
40 u32 flush_domains)
Eric Anholt62fdfea2010-05-21 13:26:39 -070041{
Eric Anholt62fdfea2010-05-21 13:26:39 -070042#if WATCH_EXEC
43 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
44 invalidate_domains, flush_domains);
45#endif
Zou Nan hai8187a2b2010-05-21 09:08:55 +080046 u32 cmd;
47 trace_i915_gem_request_flush(dev, ring->next_seqno,
Eric Anholt62fdfea2010-05-21 13:26:39 -070048 invalidate_domains, flush_domains);
49
Eric Anholt62fdfea2010-05-21 13:26:39 -070050 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
51 /*
52 * read/write caches:
53 *
54 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
55 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
56 * also flushed at 2d versus 3d pipeline switches.
57 *
58 * read-only caches:
59 *
60 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
61 * MI_READ_FLUSH is set, and is always flushed on 965.
62 *
63 * I915_GEM_DOMAIN_COMMAND may not exist?
64 *
65 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
66 * invalidated when MI_EXE_FLUSH is set.
67 *
68 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
69 * invalidated with every MI_FLUSH.
70 *
71 * TLBs:
72 *
73 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
74 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
75 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
76 * are flushed at any MI_FLUSH.
77 */
78
79 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
80 if ((invalidate_domains|flush_domains) &
81 I915_GEM_DOMAIN_RENDER)
82 cmd &= ~MI_NO_WRITE_FLUSH;
83 if (!IS_I965G(dev)) {
84 /*
85 * On the 965, the sampler cache always gets flushed
86 * and this bit is reserved.
87 */
88 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
89 cmd |= MI_READ_FLUSH;
90 }
91 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
92 cmd |= MI_EXE_FLUSH;
93
94#if WATCH_EXEC
95 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
96#endif
Zou Nan haibe26a102010-06-12 17:40:24 +080097 intel_ring_begin(dev, ring, 2);
Zou Nan hai8187a2b2010-05-21 09:08:55 +080098 intel_ring_emit(dev, ring, cmd);
99 intel_ring_emit(dev, ring, MI_NOOP);
100 intel_ring_advance(dev, ring);
101 }
102}
103
104static unsigned int render_ring_get_head(struct drm_device *dev,
105 struct intel_ring_buffer *ring)
106{
107 drm_i915_private_t *dev_priv = dev->dev_private;
108 return I915_READ(PRB0_HEAD) & HEAD_ADDR;
109}
110
111static unsigned int render_ring_get_tail(struct drm_device *dev,
112 struct intel_ring_buffer *ring)
113{
114 drm_i915_private_t *dev_priv = dev->dev_private;
115 return I915_READ(PRB0_TAIL) & TAIL_ADDR;
116}
117
118static unsigned int render_ring_get_active_head(struct drm_device *dev,
119 struct intel_ring_buffer *ring)
120{
121 drm_i915_private_t *dev_priv = dev->dev_private;
122 u32 acthd_reg = IS_I965G(dev) ? ACTHD_I965 : ACTHD;
123
124 return I915_READ(acthd_reg);
125}
126
127static void render_ring_advance_ring(struct drm_device *dev,
128 struct intel_ring_buffer *ring)
129{
130 drm_i915_private_t *dev_priv = dev->dev_private;
131 I915_WRITE(PRB0_TAIL, ring->tail);
132}
133
134static int init_ring_common(struct drm_device *dev,
135 struct intel_ring_buffer *ring)
136{
137 u32 head;
138 drm_i915_private_t *dev_priv = dev->dev_private;
139 struct drm_i915_gem_object *obj_priv;
140 obj_priv = to_intel_bo(ring->gem_object);
141
142 /* Stop the ring if it's running. */
143 I915_WRITE(ring->regs.ctl, 0);
144 I915_WRITE(ring->regs.head, 0);
145 I915_WRITE(ring->regs.tail, 0);
146
147 /* Initialize the ring. */
148 I915_WRITE(ring->regs.start, obj_priv->gtt_offset);
149 head = ring->get_head(dev, ring);
150
151 /* G45 ring initialization fails to reset head to zero */
152 if (head != 0) {
153 DRM_ERROR("%s head not reset to zero "
154 "ctl %08x head %08x tail %08x start %08x\n",
155 ring->name,
156 I915_READ(ring->regs.ctl),
157 I915_READ(ring->regs.head),
158 I915_READ(ring->regs.tail),
159 I915_READ(ring->regs.start));
160
161 I915_WRITE(ring->regs.head, 0);
162
163 DRM_ERROR("%s head forced to zero "
164 "ctl %08x head %08x tail %08x start %08x\n",
165 ring->name,
166 I915_READ(ring->regs.ctl),
167 I915_READ(ring->regs.head),
168 I915_READ(ring->regs.tail),
169 I915_READ(ring->regs.start));
Eric Anholt62fdfea2010-05-21 13:26:39 -0700170 }
171
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800172 I915_WRITE(ring->regs.ctl,
173 ((ring->gem_object->size - PAGE_SIZE) & RING_NR_PAGES)
174 | RING_NO_REPORT | RING_VALID);
175
176 head = I915_READ(ring->regs.head) & HEAD_ADDR;
177 /* If the head is still not zero, the ring is dead */
178 if (head != 0) {
179 DRM_ERROR("%s initialization failed "
180 "ctl %08x head %08x tail %08x start %08x\n",
181 ring->name,
182 I915_READ(ring->regs.ctl),
183 I915_READ(ring->regs.head),
184 I915_READ(ring->regs.tail),
185 I915_READ(ring->regs.start));
186 return -EIO;
187 }
188
189 if (!drm_core_check_feature(dev, DRIVER_MODESET))
190 i915_kernel_lost_context(dev);
191 else {
192 ring->head = ring->get_head(dev, ring);
193 ring->tail = ring->get_tail(dev, ring);
194 ring->space = ring->head - (ring->tail + 8);
195 if (ring->space < 0)
196 ring->space += ring->size;
197 }
198 return 0;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700199}
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800200
201static int init_render_ring(struct drm_device *dev,
202 struct intel_ring_buffer *ring)
203{
204 drm_i915_private_t *dev_priv = dev->dev_private;
205 int ret = init_ring_common(dev, ring);
206 if (IS_I9XX(dev) && !IS_GEN3(dev)) {
207 I915_WRITE(MI_MODE,
208 (VS_TIMER_DISPATCH) << 16 | VS_TIMER_DISPATCH);
209 }
210 return ret;
211}
212
Eric Anholt62fdfea2010-05-21 13:26:39 -0700213#define PIPE_CONTROL_FLUSH(addr) \
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800214do { \
Eric Anholt62fdfea2010-05-21 13:26:39 -0700215 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
Zhenyu Wangca764822010-05-27 10:26:42 +0800216 PIPE_CONTROL_DEPTH_STALL | 2); \
Eric Anholt62fdfea2010-05-21 13:26:39 -0700217 OUT_RING(addr | PIPE_CONTROL_GLOBAL_GTT); \
218 OUT_RING(0); \
219 OUT_RING(0); \
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800220} while (0)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700221
222/**
223 * Creates a new sequence number, emitting a write of it to the status page
224 * plus an interrupt, which will trigger i915_user_interrupt_handler.
225 *
226 * Must be called with struct_lock held.
227 *
228 * Returned sequence numbers are nonzero on success.
229 */
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800230static u32
231render_ring_add_request(struct drm_device *dev,
232 struct intel_ring_buffer *ring,
233 struct drm_file *file_priv,
234 u32 flush_domains)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700235{
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800236 u32 seqno;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700237 drm_i915_private_t *dev_priv = dev->dev_private;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800238 seqno = intel_ring_get_seqno(dev, ring);
Zhenyu Wangca764822010-05-27 10:26:42 +0800239
240 if (IS_GEN6(dev)) {
241 BEGIN_LP_RING(6);
242 OUT_RING(GFX_OP_PIPE_CONTROL | 3);
243 OUT_RING(PIPE_CONTROL_QW_WRITE |
244 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_IS_FLUSH |
245 PIPE_CONTROL_NOTIFY);
246 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
247 OUT_RING(seqno);
248 OUT_RING(0);
249 OUT_RING(0);
250 ADVANCE_LP_RING();
251 } else if (HAS_PIPE_CONTROL(dev)) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700252 u32 scratch_addr = dev_priv->seqno_gfx_addr + 128;
253
254 /*
255 * Workaround qword write incoherence by flushing the
256 * PIPE_NOTIFY buffers out to memory before requesting
257 * an interrupt.
258 */
259 BEGIN_LP_RING(32);
260 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
261 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
262 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
263 OUT_RING(seqno);
264 OUT_RING(0);
265 PIPE_CONTROL_FLUSH(scratch_addr);
266 scratch_addr += 128; /* write to separate cachelines */
267 PIPE_CONTROL_FLUSH(scratch_addr);
268 scratch_addr += 128;
269 PIPE_CONTROL_FLUSH(scratch_addr);
270 scratch_addr += 128;
271 PIPE_CONTROL_FLUSH(scratch_addr);
272 scratch_addr += 128;
273 PIPE_CONTROL_FLUSH(scratch_addr);
274 scratch_addr += 128;
275 PIPE_CONTROL_FLUSH(scratch_addr);
276 OUT_RING(GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
277 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
278 PIPE_CONTROL_NOTIFY);
279 OUT_RING(dev_priv->seqno_gfx_addr | PIPE_CONTROL_GLOBAL_GTT);
280 OUT_RING(seqno);
281 OUT_RING(0);
282 ADVANCE_LP_RING();
283 } else {
284 BEGIN_LP_RING(4);
285 OUT_RING(MI_STORE_DWORD_INDEX);
286 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
287 OUT_RING(seqno);
288
289 OUT_RING(MI_USER_INTERRUPT);
290 ADVANCE_LP_RING();
291 }
292 return seqno;
293}
294
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800295static u32
296render_ring_get_gem_seqno(struct drm_device *dev,
297 struct intel_ring_buffer *ring)
298{
299 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
300 if (HAS_PIPE_CONTROL(dev))
301 return ((volatile u32 *)(dev_priv->seqno_page))[0];
302 else
303 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
304}
305
306static void
307render_ring_get_user_irq(struct drm_device *dev,
308 struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700309{
310 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
311 unsigned long irqflags;
312
313 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800314 if (dev->irq_enabled && (++ring->user_irq_refcount == 1)) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700315 if (HAS_PCH_SPLIT(dev))
316 ironlake_enable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
317 else
318 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
319 }
320 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
321}
322
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800323static void
324render_ring_put_user_irq(struct drm_device *dev,
325 struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700326{
327 drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
328 unsigned long irqflags;
329
330 spin_lock_irqsave(&dev_priv->user_irq_lock, irqflags);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800331 BUG_ON(dev->irq_enabled && ring->user_irq_refcount <= 0);
332 if (dev->irq_enabled && (--ring->user_irq_refcount == 0)) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700333 if (HAS_PCH_SPLIT(dev))
334 ironlake_disable_graphics_irq(dev_priv, GT_PIPE_NOTIFY);
335 else
336 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
337 }
338 spin_unlock_irqrestore(&dev_priv->user_irq_lock, irqflags);
339}
340
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800341static void render_setup_status_page(struct drm_device *dev,
342 struct intel_ring_buffer *ring)
343{
344 drm_i915_private_t *dev_priv = dev->dev_private;
345 if (IS_GEN6(dev)) {
346 I915_WRITE(HWS_PGA_GEN6, ring->status_page.gfx_addr);
347 I915_READ(HWS_PGA_GEN6); /* posting read */
348 } else {
349 I915_WRITE(HWS_PGA, ring->status_page.gfx_addr);
350 I915_READ(HWS_PGA); /* posting read */
351 }
352
353}
354
Zou Nan haid1b851f2010-05-21 09:08:57 +0800355void
356bsd_ring_flush(struct drm_device *dev,
357 struct intel_ring_buffer *ring,
358 u32 invalidate_domains,
359 u32 flush_domains)
360{
Zou Nan haibe26a102010-06-12 17:40:24 +0800361 intel_ring_begin(dev, ring, 2);
Zou Nan haid1b851f2010-05-21 09:08:57 +0800362 intel_ring_emit(dev, ring, MI_FLUSH);
363 intel_ring_emit(dev, ring, MI_NOOP);
364 intel_ring_advance(dev, ring);
365}
366
367static inline unsigned int bsd_ring_get_head(struct drm_device *dev,
368 struct intel_ring_buffer *ring)
369{
370 drm_i915_private_t *dev_priv = dev->dev_private;
371 return I915_READ(BSD_RING_HEAD) & HEAD_ADDR;
372}
373
374static inline unsigned int bsd_ring_get_tail(struct drm_device *dev,
375 struct intel_ring_buffer *ring)
376{
377 drm_i915_private_t *dev_priv = dev->dev_private;
378 return I915_READ(BSD_RING_TAIL) & TAIL_ADDR;
379}
380
381static inline unsigned int bsd_ring_get_active_head(struct drm_device *dev,
382 struct intel_ring_buffer *ring)
383{
384 drm_i915_private_t *dev_priv = dev->dev_private;
385 return I915_READ(BSD_RING_ACTHD);
386}
387
388static inline void bsd_ring_advance_ring(struct drm_device *dev,
389 struct intel_ring_buffer *ring)
390{
391 drm_i915_private_t *dev_priv = dev->dev_private;
392 I915_WRITE(BSD_RING_TAIL, ring->tail);
393}
394
395static int init_bsd_ring(struct drm_device *dev,
396 struct intel_ring_buffer *ring)
397{
398 return init_ring_common(dev, ring);
399}
400
401static u32
402bsd_ring_add_request(struct drm_device *dev,
403 struct intel_ring_buffer *ring,
404 struct drm_file *file_priv,
405 u32 flush_domains)
406{
407 u32 seqno;
408 seqno = intel_ring_get_seqno(dev, ring);
409 intel_ring_begin(dev, ring, 4);
410 intel_ring_emit(dev, ring, MI_STORE_DWORD_INDEX);
411 intel_ring_emit(dev, ring,
412 I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
413 intel_ring_emit(dev, ring, seqno);
414 intel_ring_emit(dev, ring, MI_USER_INTERRUPT);
415 intel_ring_advance(dev, ring);
416
417 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
418
419 return seqno;
420}
421
422static void bsd_setup_status_page(struct drm_device *dev,
423 struct intel_ring_buffer *ring)
424{
425 drm_i915_private_t *dev_priv = dev->dev_private;
426 I915_WRITE(BSD_HWS_PGA, ring->status_page.gfx_addr);
427 I915_READ(BSD_HWS_PGA);
428}
429
430static void
431bsd_ring_get_user_irq(struct drm_device *dev,
432 struct intel_ring_buffer *ring)
433{
434 /* do nothing */
435}
436static void
437bsd_ring_put_user_irq(struct drm_device *dev,
438 struct intel_ring_buffer *ring)
439{
440 /* do nothing */
441}
442
443static u32
444bsd_ring_get_gem_seqno(struct drm_device *dev,
445 struct intel_ring_buffer *ring)
446{
447 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
448}
449
450static int
451bsd_ring_dispatch_gem_execbuffer(struct drm_device *dev,
452 struct intel_ring_buffer *ring,
453 struct drm_i915_gem_execbuffer2 *exec,
454 struct drm_clip_rect *cliprects,
455 uint64_t exec_offset)
456{
457 uint32_t exec_start;
458 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
459 intel_ring_begin(dev, ring, 2);
460 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START |
461 (2 << 6) | MI_BATCH_NON_SECURE_I965);
462 intel_ring_emit(dev, ring, exec_start);
463 intel_ring_advance(dev, ring);
464 return 0;
465}
466
467
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800468static int
469render_ring_dispatch_gem_execbuffer(struct drm_device *dev,
470 struct intel_ring_buffer *ring,
471 struct drm_i915_gem_execbuffer2 *exec,
472 struct drm_clip_rect *cliprects,
473 uint64_t exec_offset)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700474{
475 drm_i915_private_t *dev_priv = dev->dev_private;
476 int nbox = exec->num_cliprects;
477 int i = 0, count;
478 uint32_t exec_start, exec_len;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700479 exec_start = (uint32_t) exec_offset + exec->batch_start_offset;
480 exec_len = (uint32_t) exec->batch_len;
481
482 trace_i915_gem_request_submit(dev, dev_priv->mm.next_gem_seqno + 1);
483
484 count = nbox ? nbox : 1;
485
486 for (i = 0; i < count; i++) {
487 if (i < nbox) {
488 int ret = i915_emit_box(dev, cliprects, i,
489 exec->DR1, exec->DR4);
490 if (ret)
491 return ret;
492 }
493
494 if (IS_I830(dev) || IS_845G(dev)) {
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800495 intel_ring_begin(dev, ring, 4);
496 intel_ring_emit(dev, ring, MI_BATCH_BUFFER);
497 intel_ring_emit(dev, ring,
498 exec_start | MI_BATCH_NON_SECURE);
499 intel_ring_emit(dev, ring, exec_start + exec_len - 4);
500 intel_ring_emit(dev, ring, 0);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700501 } else {
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800502 intel_ring_begin(dev, ring, 4);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700503 if (IS_I965G(dev)) {
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800504 intel_ring_emit(dev, ring,
505 MI_BATCH_BUFFER_START | (2 << 6)
506 | MI_BATCH_NON_SECURE_I965);
507 intel_ring_emit(dev, ring, exec_start);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700508 } else {
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800509 intel_ring_emit(dev, ring, MI_BATCH_BUFFER_START
510 | (2 << 6));
511 intel_ring_emit(dev, ring, exec_start |
512 MI_BATCH_NON_SECURE);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700513 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700514 }
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800515 intel_ring_advance(dev, ring);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700516 }
517
518 /* XXX breadcrumb */
519 return 0;
520}
521
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800522static void cleanup_status_page(struct drm_device *dev,
523 struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700524{
525 drm_i915_private_t *dev_priv = dev->dev_private;
526 struct drm_gem_object *obj;
527 struct drm_i915_gem_object *obj_priv;
528
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800529 obj = ring->status_page.obj;
530 if (obj == NULL)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700531 return;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700532 obj_priv = to_intel_bo(obj);
533
534 kunmap(obj_priv->pages[0]);
535 i915_gem_object_unpin(obj);
536 drm_gem_object_unreference(obj);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800537 ring->status_page.obj = NULL;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700538
539 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
Eric Anholt62fdfea2010-05-21 13:26:39 -0700540}
541
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800542static int init_status_page(struct drm_device *dev,
543 struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700544{
545 drm_i915_private_t *dev_priv = dev->dev_private;
546 struct drm_gem_object *obj;
547 struct drm_i915_gem_object *obj_priv;
548 int ret;
549
Eric Anholt62fdfea2010-05-21 13:26:39 -0700550 obj = i915_gem_alloc_object(dev, 4096);
551 if (obj == NULL) {
552 DRM_ERROR("Failed to allocate status page\n");
553 ret = -ENOMEM;
554 goto err;
555 }
556 obj_priv = to_intel_bo(obj);
557 obj_priv->agp_type = AGP_USER_CACHED_MEMORY;
558
559 ret = i915_gem_object_pin(obj, 4096);
560 if (ret != 0) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700561 goto err_unref;
562 }
563
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800564 ring->status_page.gfx_addr = obj_priv->gtt_offset;
565 ring->status_page.page_addr = kmap(obj_priv->pages[0]);
566 if (ring->status_page.page_addr == NULL) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700567 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
Eric Anholt62fdfea2010-05-21 13:26:39 -0700568 goto err_unpin;
569 }
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800570 ring->status_page.obj = obj;
571 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700572
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800573 ring->setup_status_page(dev, ring);
574 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
575 ring->name, ring->status_page.gfx_addr);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700576
577 return 0;
578
579err_unpin:
580 i915_gem_object_unpin(obj);
581err_unref:
582 drm_gem_object_unreference(obj);
583err:
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800584 return ret;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700585}
586
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800587
588int intel_init_ring_buffer(struct drm_device *dev,
589 struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700590{
Eric Anholt62fdfea2010-05-21 13:26:39 -0700591 int ret;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800592 struct drm_i915_gem_object *obj_priv;
593 struct drm_gem_object *obj;
594 ring->dev = dev;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700595
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800596 if (I915_NEED_GFX_HWS(dev)) {
597 ret = init_status_page(dev, ring);
598 if (ret)
599 return ret;
600 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700601
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800602 obj = i915_gem_alloc_object(dev, ring->size);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700603 if (obj == NULL) {
604 DRM_ERROR("Failed to allocate ringbuffer\n");
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800605 ret = -ENOMEM;
606 goto cleanup;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700607 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700608
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800609 ring->gem_object = obj;
610
611 ret = i915_gem_object_pin(obj, ring->alignment);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700612 if (ret != 0) {
613 drm_gem_object_unreference(obj);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800614 goto cleanup;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700615 }
616
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800617 obj_priv = to_intel_bo(obj);
618 ring->map.size = ring->size;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700619 ring->map.offset = dev->agp->base + obj_priv->gtt_offset;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700620 ring->map.type = 0;
621 ring->map.flags = 0;
622 ring->map.mtrr = 0;
623
624 drm_core_ioremap_wc(&ring->map, dev);
625 if (ring->map.handle == NULL) {
626 DRM_ERROR("Failed to map ringbuffer.\n");
Eric Anholt62fdfea2010-05-21 13:26:39 -0700627 i915_gem_object_unpin(obj);
628 drm_gem_object_unreference(obj);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800629 ret = -EINVAL;
630 goto cleanup;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700631 }
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800632
Eric Anholt62fdfea2010-05-21 13:26:39 -0700633 ring->virtual_start = ring->map.handle;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800634 ret = ring->init(dev, ring);
635 if (ret != 0) {
636 intel_cleanup_ring_buffer(dev, ring);
637 return ret;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700638 }
639
Eric Anholt62fdfea2010-05-21 13:26:39 -0700640 if (!drm_core_check_feature(dev, DRIVER_MODESET))
641 i915_kernel_lost_context(dev);
642 else {
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800643 ring->head = ring->get_head(dev, ring);
644 ring->tail = ring->get_tail(dev, ring);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700645 ring->space = ring->head - (ring->tail + 8);
646 if (ring->space < 0)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800647 ring->space += ring->size;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700648 }
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800649 INIT_LIST_HEAD(&ring->active_list);
650 INIT_LIST_HEAD(&ring->request_list);
651 return ret;
652cleanup:
653 cleanup_status_page(dev, ring);
654 return ret;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700655}
656
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800657void intel_cleanup_ring_buffer(struct drm_device *dev,
658 struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700659{
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800660 if (ring->gem_object == NULL)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700661 return;
662
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800663 drm_core_ioremapfree(&ring->map, dev);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700664
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800665 i915_gem_object_unpin(ring->gem_object);
666 drm_gem_object_unreference(ring->gem_object);
667 ring->gem_object = NULL;
668 cleanup_status_page(dev, ring);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700669}
670
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800671int intel_wrap_ring_buffer(struct drm_device *dev,
672 struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700673{
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800674 unsigned int *virt;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700675 int rem;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800676 rem = ring->size - ring->tail;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700677
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800678 if (ring->space < rem) {
679 int ret = intel_wait_ring_buffer(dev, ring, rem);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700680 if (ret)
681 return ret;
682 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700683
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800684 virt = (unsigned int *)(ring->virtual_start + ring->tail);
Chris Wilson1741dd42010-08-04 15:18:12 +0100685 rem /= 8;
686 while (rem--) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700687 *virt++ = MI_NOOP;
Chris Wilson1741dd42010-08-04 15:18:12 +0100688 *virt++ = MI_NOOP;
689 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700690
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800691 ring->tail = 0;
Chris Wilson43ed3402010-07-01 17:53:00 +0100692 ring->space = ring->head - 8;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700693
694 return 0;
695}
696
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800697int intel_wait_ring_buffer(struct drm_device *dev,
698 struct intel_ring_buffer *ring, int n)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700699{
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800700 unsigned long end;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700701
702 trace_i915_ring_wait_begin (dev);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800703 end = jiffies + 3 * HZ;
704 do {
705 ring->head = ring->get_head(dev, ring);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700706 ring->space = ring->head - (ring->tail + 8);
707 if (ring->space < 0)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800708 ring->space += ring->size;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700709 if (ring->space >= n) {
710 trace_i915_ring_wait_end (dev);
711 return 0;
712 }
713
714 if (dev->primary->master) {
715 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
716 if (master_priv->sarea_priv)
717 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
718 }
Zou Nan haid1b851f2010-05-21 09:08:57 +0800719
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800720 yield();
721 } while (!time_after(jiffies, end));
Eric Anholt62fdfea2010-05-21 13:26:39 -0700722 trace_i915_ring_wait_end (dev);
723 return -EBUSY;
724}
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800725
726void intel_ring_begin(struct drm_device *dev,
Zou Nan haibe26a102010-06-12 17:40:24 +0800727 struct intel_ring_buffer *ring, int num_dwords)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800728{
Zou Nan haibe26a102010-06-12 17:40:24 +0800729 int n = 4*num_dwords;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800730 if (unlikely(ring->tail + n > ring->size))
731 intel_wrap_ring_buffer(dev, ring);
732 if (unlikely(ring->space < n))
733 intel_wait_ring_buffer(dev, ring, n);
Chris Wilsond97ed332010-08-04 15:18:13 +0100734
735 ring->space -= n;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800736}
737
738void intel_ring_emit(struct drm_device *dev,
739 struct intel_ring_buffer *ring, unsigned int data)
740{
741 unsigned int *virt = ring->virtual_start + ring->tail;
742 *virt = data;
743 ring->tail += 4;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800744}
745
746void intel_ring_advance(struct drm_device *dev,
747 struct intel_ring_buffer *ring)
748{
Chris Wilsond97ed332010-08-04 15:18:13 +0100749 ring->tail &= ring->size - 1;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800750 ring->advance_ring(dev, ring);
751}
752
753void intel_fill_struct(struct drm_device *dev,
754 struct intel_ring_buffer *ring,
755 void *data,
756 unsigned int len)
757{
758 unsigned int *virt = ring->virtual_start + ring->tail;
759 BUG_ON((len&~(4-1)) != 0);
Zou Nan haibe26a102010-06-12 17:40:24 +0800760 intel_ring_begin(dev, ring, len/4);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800761 memcpy(virt, data, len);
762 ring->tail += len;
763 ring->tail &= ring->size - 1;
764 ring->space -= len;
765 intel_ring_advance(dev, ring);
766}
767
768u32 intel_ring_get_seqno(struct drm_device *dev,
769 struct intel_ring_buffer *ring)
770{
771 u32 seqno;
772 seqno = ring->next_seqno;
773
774 /* reserve 0 for non-seqno */
775 if (++ring->next_seqno == 0)
776 ring->next_seqno = 1;
777 return seqno;
778}
779
780struct intel_ring_buffer render_ring = {
781 .name = "render ring",
782 .regs = {
783 .ctl = PRB0_CTL,
784 .head = PRB0_HEAD,
785 .tail = PRB0_TAIL,
786 .start = PRB0_START
787 },
788 .ring_flag = I915_EXEC_RENDER,
789 .size = 32 * PAGE_SIZE,
790 .alignment = PAGE_SIZE,
791 .virtual_start = NULL,
792 .dev = NULL,
793 .gem_object = NULL,
794 .head = 0,
795 .tail = 0,
796 .space = 0,
797 .next_seqno = 1,
798 .user_irq_refcount = 0,
799 .irq_gem_seqno = 0,
800 .waiting_gem_seqno = 0,
801 .setup_status_page = render_setup_status_page,
802 .init = init_render_ring,
803 .get_head = render_ring_get_head,
804 .get_tail = render_ring_get_tail,
805 .get_active_head = render_ring_get_active_head,
806 .advance_ring = render_ring_advance_ring,
807 .flush = render_ring_flush,
808 .add_request = render_ring_add_request,
809 .get_gem_seqno = render_ring_get_gem_seqno,
810 .user_irq_get = render_ring_get_user_irq,
811 .user_irq_put = render_ring_put_user_irq,
812 .dispatch_gem_execbuffer = render_ring_dispatch_gem_execbuffer,
813 .status_page = {NULL, 0, NULL},
814 .map = {0,}
815};
Zou Nan haid1b851f2010-05-21 09:08:57 +0800816
817/* ring buffer for bit-stream decoder */
818
819struct intel_ring_buffer bsd_ring = {
820 .name = "bsd ring",
821 .regs = {
822 .ctl = BSD_RING_CTL,
823 .head = BSD_RING_HEAD,
824 .tail = BSD_RING_TAIL,
825 .start = BSD_RING_START
826 },
827 .ring_flag = I915_EXEC_BSD,
828 .size = 32 * PAGE_SIZE,
829 .alignment = PAGE_SIZE,
830 .virtual_start = NULL,
831 .dev = NULL,
832 .gem_object = NULL,
833 .head = 0,
834 .tail = 0,
835 .space = 0,
836 .next_seqno = 1,
837 .user_irq_refcount = 0,
838 .irq_gem_seqno = 0,
839 .waiting_gem_seqno = 0,
840 .setup_status_page = bsd_setup_status_page,
841 .init = init_bsd_ring,
842 .get_head = bsd_ring_get_head,
843 .get_tail = bsd_ring_get_tail,
844 .get_active_head = bsd_ring_get_active_head,
845 .advance_ring = bsd_ring_advance_ring,
846 .flush = bsd_ring_flush,
847 .add_request = bsd_ring_add_request,
848 .get_gem_seqno = bsd_ring_get_gem_seqno,
849 .user_irq_get = bsd_ring_get_user_irq,
850 .user_irq_put = bsd_ring_put_user_irq,
851 .dispatch_gem_execbuffer = bsd_ring_dispatch_gem_execbuffer,
852 .status_page = {NULL, 0, NULL},
853 .map = {0,}
854};