blob: 03e3370725179bd14785aef19bf83f5c13c285ee [file] [log] [blame]
Eric Anholt62fdfea2010-05-21 13:26:39 -07001/*
2 * Copyright © 2008-2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 * Zou Nan hai <nanhai.zou@intel.com>
26 * Xiang Hai hao<haihao.xiang@intel.com>
27 *
28 */
29
30#include "drmP.h"
31#include "drm.h"
Eric Anholt62fdfea2010-05-21 13:26:39 -070032#include "i915_drv.h"
Zou Nan hai8187a2b2010-05-21 09:08:55 +080033#include "i915_drm.h"
Eric Anholt62fdfea2010-05-21 13:26:39 -070034#include "i915_trace.h"
Xiang, Haihao881f47b2010-09-19 14:40:43 +010035#include "intel_drv.h"
Eric Anholt62fdfea2010-05-21 13:26:39 -070036
Chris Wilson6f392d52010-08-07 11:01:22 +010037static u32 i915_gem_get_seqno(struct drm_device *dev)
38{
39 drm_i915_private_t *dev_priv = dev->dev_private;
40 u32 seqno;
41
42 seqno = dev_priv->next_seqno;
43
44 /* reserve 0 for non-seqno */
45 if (++dev_priv->next_seqno == 0)
46 dev_priv->next_seqno = 1;
47
48 return seqno;
49}
50
Chris Wilsonb72f3ac2011-01-04 17:34:02 +000051static int
Chris Wilson78501ea2010-10-27 12:18:21 +010052render_ring_flush(struct intel_ring_buffer *ring,
Chris Wilsonab6f8e32010-09-19 17:53:44 +010053 u32 invalidate_domains,
54 u32 flush_domains)
Eric Anholt62fdfea2010-05-21 13:26:39 -070055{
Chris Wilson78501ea2010-10-27 12:18:21 +010056 struct drm_device *dev = ring->dev;
Chris Wilson6f392d52010-08-07 11:01:22 +010057 drm_i915_private_t *dev_priv = dev->dev_private;
58 u32 cmd;
Chris Wilsonb72f3ac2011-01-04 17:34:02 +000059 int ret;
Chris Wilson6f392d52010-08-07 11:01:22 +010060
Eric Anholt62fdfea2010-05-21 13:26:39 -070061#if WATCH_EXEC
62 DRM_INFO("%s: invalidate %08x flush %08x\n", __func__,
63 invalidate_domains, flush_domains);
64#endif
Chris Wilson6f392d52010-08-07 11:01:22 +010065
66 trace_i915_gem_request_flush(dev, dev_priv->next_seqno,
Eric Anholt62fdfea2010-05-21 13:26:39 -070067 invalidate_domains, flush_domains);
68
Eric Anholt62fdfea2010-05-21 13:26:39 -070069 if ((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) {
70 /*
71 * read/write caches:
72 *
73 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
74 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
75 * also flushed at 2d versus 3d pipeline switches.
76 *
77 * read-only caches:
78 *
79 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
80 * MI_READ_FLUSH is set, and is always flushed on 965.
81 *
82 * I915_GEM_DOMAIN_COMMAND may not exist?
83 *
84 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
85 * invalidated when MI_EXE_FLUSH is set.
86 *
87 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
88 * invalidated with every MI_FLUSH.
89 *
90 * TLBs:
91 *
92 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
93 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
94 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
95 * are flushed at any MI_FLUSH.
96 */
97
98 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
99 if ((invalidate_domains|flush_domains) &
100 I915_GEM_DOMAIN_RENDER)
101 cmd &= ~MI_NO_WRITE_FLUSH;
Chris Wilsona6c45cf2010-09-17 00:32:17 +0100102 if (INTEL_INFO(dev)->gen < 4) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700103 /*
104 * On the 965, the sampler cache always gets flushed
105 * and this bit is reserved.
106 */
107 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
108 cmd |= MI_READ_FLUSH;
109 }
110 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
111 cmd |= MI_EXE_FLUSH;
112
Chris Wilson70eac332010-11-30 14:07:47 +0000113 if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
114 (IS_G4X(dev) || IS_GEN5(dev)))
115 cmd |= MI_INVALIDATE_ISP;
116
Eric Anholt62fdfea2010-05-21 13:26:39 -0700117#if WATCH_EXEC
118 DRM_INFO("%s: queue flush %08x to ring\n", __func__, cmd);
119#endif
Chris Wilsonb72f3ac2011-01-04 17:34:02 +0000120 ret = intel_ring_begin(ring, 2);
121 if (ret)
122 return ret;
123
124 intel_ring_emit(ring, cmd);
125 intel_ring_emit(ring, MI_NOOP);
126 intel_ring_advance(ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800127 }
Chris Wilsonb72f3ac2011-01-04 17:34:02 +0000128
129 return 0;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800130}
131
Chris Wilson78501ea2010-10-27 12:18:21 +0100132static void ring_write_tail(struct intel_ring_buffer *ring,
Chris Wilson297b0c52010-10-22 17:02:41 +0100133 u32 value)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800134{
Chris Wilson78501ea2010-10-27 12:18:21 +0100135 drm_i915_private_t *dev_priv = ring->dev->dev_private;
Chris Wilson297b0c52010-10-22 17:02:41 +0100136 I915_WRITE_TAIL(ring, value);
Xiang, Haihaod46eefa2010-09-16 10:43:12 +0800137}
138
Chris Wilson78501ea2010-10-27 12:18:21 +0100139u32 intel_ring_get_active_head(struct intel_ring_buffer *ring)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800140{
Chris Wilson78501ea2010-10-27 12:18:21 +0100141 drm_i915_private_t *dev_priv = ring->dev->dev_private;
142 u32 acthd_reg = INTEL_INFO(ring->dev)->gen >= 4 ?
Daniel Vetter3d281d82010-09-24 21:14:22 +0200143 RING_ACTHD(ring->mmio_base) : ACTHD;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800144
145 return I915_READ(acthd_reg);
146}
147
Chris Wilson78501ea2010-10-27 12:18:21 +0100148static int init_ring_common(struct intel_ring_buffer *ring)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800149{
Chris Wilson78501ea2010-10-27 12:18:21 +0100150 drm_i915_private_t *dev_priv = ring->dev->dev_private;
Chris Wilson05394f32010-11-08 19:18:58 +0000151 struct drm_i915_gem_object *obj = ring->obj;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800152 u32 head;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800153
154 /* Stop the ring if it's running. */
Daniel Vetter7f2ab692010-08-02 17:06:59 +0200155 I915_WRITE_CTL(ring, 0);
Daniel Vetter570ef602010-08-02 17:06:23 +0200156 I915_WRITE_HEAD(ring, 0);
Chris Wilson78501ea2010-10-27 12:18:21 +0100157 ring->write_tail(ring, 0);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800158
159 /* Initialize the ring. */
Chris Wilson05394f32010-11-08 19:18:58 +0000160 I915_WRITE_START(ring, obj->gtt_offset);
Daniel Vetter570ef602010-08-02 17:06:23 +0200161 head = I915_READ_HEAD(ring) & HEAD_ADDR;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800162
163 /* G45 ring initialization fails to reset head to zero */
164 if (head != 0) {
Chris Wilson6fd0d562010-12-05 20:42:33 +0000165 DRM_DEBUG_KMS("%s head not reset to zero "
166 "ctl %08x head %08x tail %08x start %08x\n",
167 ring->name,
168 I915_READ_CTL(ring),
169 I915_READ_HEAD(ring),
170 I915_READ_TAIL(ring),
171 I915_READ_START(ring));
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800172
Daniel Vetter570ef602010-08-02 17:06:23 +0200173 I915_WRITE_HEAD(ring, 0);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800174
Chris Wilson6fd0d562010-12-05 20:42:33 +0000175 if (I915_READ_HEAD(ring) & HEAD_ADDR) {
176 DRM_ERROR("failed to set %s head to zero "
177 "ctl %08x head %08x tail %08x start %08x\n",
178 ring->name,
179 I915_READ_CTL(ring),
180 I915_READ_HEAD(ring),
181 I915_READ_TAIL(ring),
182 I915_READ_START(ring));
183 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700184 }
185
Daniel Vetter7f2ab692010-08-02 17:06:59 +0200186 I915_WRITE_CTL(ring,
Chris Wilsonae69b422010-11-07 11:45:52 +0000187 ((ring->size - PAGE_SIZE) & RING_NR_PAGES)
Chris Wilson6aa56062010-10-29 21:44:37 +0100188 | RING_REPORT_64K | RING_VALID);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800189
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800190 /* If the head is still not zero, the ring is dead */
Chris Wilson176f28e2010-10-28 11:18:07 +0100191 if ((I915_READ_CTL(ring) & RING_VALID) == 0 ||
Chris Wilson05394f32010-11-08 19:18:58 +0000192 I915_READ_START(ring) != obj->gtt_offset ||
Chris Wilson176f28e2010-10-28 11:18:07 +0100193 (I915_READ_HEAD(ring) & HEAD_ADDR) != 0) {
Chris Wilsone74cfed2010-11-09 10:16:56 +0000194 DRM_ERROR("%s initialization failed "
195 "ctl %08x head %08x tail %08x start %08x\n",
196 ring->name,
197 I915_READ_CTL(ring),
198 I915_READ_HEAD(ring),
199 I915_READ_TAIL(ring),
200 I915_READ_START(ring));
201 return -EIO;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800202 }
203
Chris Wilson78501ea2010-10-27 12:18:21 +0100204 if (!drm_core_check_feature(ring->dev, DRIVER_MODESET))
205 i915_kernel_lost_context(ring->dev);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800206 else {
Daniel Vetter570ef602010-08-02 17:06:23 +0200207 ring->head = I915_READ_HEAD(ring) & HEAD_ADDR;
Daniel Vetter870e86d2010-08-02 16:29:44 +0200208 ring->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800209 ring->space = ring->head - (ring->tail + 8);
210 if (ring->space < 0)
211 ring->space += ring->size;
212 }
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000213
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800214 return 0;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700215}
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800216
Chris Wilsonc6df5412010-12-15 09:56:50 +0000217/*
218 * 965+ support PIPE_CONTROL commands, which provide finer grained control
219 * over cache flushing.
220 */
221struct pipe_control {
222 struct drm_i915_gem_object *obj;
223 volatile u32 *cpu_page;
224 u32 gtt_offset;
225};
226
227static int
228init_pipe_control(struct intel_ring_buffer *ring)
229{
230 struct pipe_control *pc;
231 struct drm_i915_gem_object *obj;
232 int ret;
233
234 if (ring->private)
235 return 0;
236
237 pc = kmalloc(sizeof(*pc), GFP_KERNEL);
238 if (!pc)
239 return -ENOMEM;
240
241 obj = i915_gem_alloc_object(ring->dev, 4096);
242 if (obj == NULL) {
243 DRM_ERROR("Failed to allocate seqno page\n");
244 ret = -ENOMEM;
245 goto err;
246 }
247 obj->agp_type = AGP_USER_CACHED_MEMORY;
248
249 ret = i915_gem_object_pin(obj, 4096, true);
250 if (ret)
251 goto err_unref;
252
253 pc->gtt_offset = obj->gtt_offset;
254 pc->cpu_page = kmap(obj->pages[0]);
255 if (pc->cpu_page == NULL)
256 goto err_unpin;
257
258 pc->obj = obj;
259 ring->private = pc;
260 return 0;
261
262err_unpin:
263 i915_gem_object_unpin(obj);
264err_unref:
265 drm_gem_object_unreference(&obj->base);
266err:
267 kfree(pc);
268 return ret;
269}
270
271static void
272cleanup_pipe_control(struct intel_ring_buffer *ring)
273{
274 struct pipe_control *pc = ring->private;
275 struct drm_i915_gem_object *obj;
276
277 if (!ring->private)
278 return;
279
280 obj = pc->obj;
281 kunmap(obj->pages[0]);
282 i915_gem_object_unpin(obj);
283 drm_gem_object_unreference(&obj->base);
284
285 kfree(pc);
286 ring->private = NULL;
287}
288
Chris Wilson78501ea2010-10-27 12:18:21 +0100289static int init_render_ring(struct intel_ring_buffer *ring)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800290{
Chris Wilson78501ea2010-10-27 12:18:21 +0100291 struct drm_device *dev = ring->dev;
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000292 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson78501ea2010-10-27 12:18:21 +0100293 int ret = init_ring_common(ring);
Zhenyu Wanga69ffdb2010-08-30 16:12:42 +0800294
Chris Wilsona6c45cf2010-09-17 00:32:17 +0100295 if (INTEL_INFO(dev)->gen > 3) {
Chris Wilson78501ea2010-10-27 12:18:21 +0100296 int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
Zhenyu Wanga69ffdb2010-08-30 16:12:42 +0800297 if (IS_GEN6(dev))
298 mode |= MI_FLUSH_ENABLE << 16 | MI_FLUSH_ENABLE;
299 I915_WRITE(MI_MODE, mode);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800300 }
Chris Wilson78501ea2010-10-27 12:18:21 +0100301
Chris Wilsonc6df5412010-12-15 09:56:50 +0000302 if (INTEL_INFO(dev)->gen >= 6) {
303 } else if (IS_GEN5(dev)) {
304 ret = init_pipe_control(ring);
305 if (ret)
306 return ret;
307 }
308
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800309 return ret;
310}
311
Chris Wilsonc6df5412010-12-15 09:56:50 +0000312static void render_ring_cleanup(struct intel_ring_buffer *ring)
313{
314 if (!ring->private)
315 return;
316
317 cleanup_pipe_control(ring);
318}
319
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000320static void
321update_semaphore(struct intel_ring_buffer *ring, int i, u32 seqno)
322{
323 struct drm_device *dev = ring->dev;
324 struct drm_i915_private *dev_priv = dev->dev_private;
325 int id;
326
327 /*
328 * cs -> 1 = vcs, 0 = bcs
329 * vcs -> 1 = bcs, 0 = cs,
330 * bcs -> 1 = cs, 0 = vcs.
331 */
332 id = ring - dev_priv->ring;
333 id += 2 - i;
334 id %= 3;
335
336 intel_ring_emit(ring,
337 MI_SEMAPHORE_MBOX |
338 MI_SEMAPHORE_REGISTER |
339 MI_SEMAPHORE_UPDATE);
340 intel_ring_emit(ring, seqno);
341 intel_ring_emit(ring,
342 RING_SYNC_0(dev_priv->ring[id].mmio_base) + 4*i);
343}
344
345static int
346gen6_add_request(struct intel_ring_buffer *ring,
347 u32 *result)
348{
349 u32 seqno;
350 int ret;
351
352 ret = intel_ring_begin(ring, 10);
353 if (ret)
354 return ret;
355
356 seqno = i915_gem_get_seqno(ring->dev);
357 update_semaphore(ring, 0, seqno);
358 update_semaphore(ring, 1, seqno);
359
360 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
361 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
362 intel_ring_emit(ring, seqno);
363 intel_ring_emit(ring, MI_USER_INTERRUPT);
364 intel_ring_advance(ring);
365
366 *result = seqno;
367 return 0;
368}
369
370int
371intel_ring_sync(struct intel_ring_buffer *ring,
372 struct intel_ring_buffer *to,
373 u32 seqno)
374{
375 int ret;
376
377 ret = intel_ring_begin(ring, 4);
378 if (ret)
379 return ret;
380
381 intel_ring_emit(ring,
382 MI_SEMAPHORE_MBOX |
383 MI_SEMAPHORE_REGISTER |
384 intel_ring_sync_index(ring, to) << 17 |
385 MI_SEMAPHORE_COMPARE);
386 intel_ring_emit(ring, seqno);
387 intel_ring_emit(ring, 0);
388 intel_ring_emit(ring, MI_NOOP);
389 intel_ring_advance(ring);
390
391 return 0;
392}
393
Chris Wilsonc6df5412010-12-15 09:56:50 +0000394#define PIPE_CONTROL_FLUSH(ring__, addr__) \
395do { \
396 intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE | \
397 PIPE_CONTROL_DEPTH_STALL | 2); \
398 intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT); \
399 intel_ring_emit(ring__, 0); \
400 intel_ring_emit(ring__, 0); \
401} while (0)
402
403static int
404pc_render_add_request(struct intel_ring_buffer *ring,
405 u32 *result)
406{
407 struct drm_device *dev = ring->dev;
408 u32 seqno = i915_gem_get_seqno(dev);
409 struct pipe_control *pc = ring->private;
410 u32 scratch_addr = pc->gtt_offset + 128;
411 int ret;
412
413 /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
414 * incoherent with writes to memory, i.e. completely fubar,
415 * so we need to use PIPE_NOTIFY instead.
416 *
417 * However, we also need to workaround the qword write
418 * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
419 * memory before requesting an interrupt.
420 */
421 ret = intel_ring_begin(ring, 32);
422 if (ret)
423 return ret;
424
425 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
426 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH);
427 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
428 intel_ring_emit(ring, seqno);
429 intel_ring_emit(ring, 0);
430 PIPE_CONTROL_FLUSH(ring, scratch_addr);
431 scratch_addr += 128; /* write to separate cachelines */
432 PIPE_CONTROL_FLUSH(ring, scratch_addr);
433 scratch_addr += 128;
434 PIPE_CONTROL_FLUSH(ring, scratch_addr);
435 scratch_addr += 128;
436 PIPE_CONTROL_FLUSH(ring, scratch_addr);
437 scratch_addr += 128;
438 PIPE_CONTROL_FLUSH(ring, scratch_addr);
439 scratch_addr += 128;
440 PIPE_CONTROL_FLUSH(ring, scratch_addr);
441 intel_ring_emit(ring, GFX_OP_PIPE_CONTROL | PIPE_CONTROL_QW_WRITE |
442 PIPE_CONTROL_WC_FLUSH | PIPE_CONTROL_TC_FLUSH |
443 PIPE_CONTROL_NOTIFY);
444 intel_ring_emit(ring, pc->gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
445 intel_ring_emit(ring, seqno);
446 intel_ring_emit(ring, 0);
447 intel_ring_advance(ring);
448
449 *result = seqno;
450 return 0;
451}
452
Chris Wilson3cce4692010-10-27 16:11:02 +0100453static int
Chris Wilson78501ea2010-10-27 12:18:21 +0100454render_ring_add_request(struct intel_ring_buffer *ring,
Chris Wilson3cce4692010-10-27 16:11:02 +0100455 u32 *result)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700456{
Chris Wilson78501ea2010-10-27 12:18:21 +0100457 struct drm_device *dev = ring->dev;
Chris Wilson3cce4692010-10-27 16:11:02 +0100458 u32 seqno = i915_gem_get_seqno(dev);
459 int ret;
Zhenyu Wangca764822010-05-27 10:26:42 +0800460
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000461 ret = intel_ring_begin(ring, 4);
462 if (ret)
463 return ret;
Chris Wilson3cce4692010-10-27 16:11:02 +0100464
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000465 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
466 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
467 intel_ring_emit(ring, seqno);
468 intel_ring_emit(ring, MI_USER_INTERRUPT);
Chris Wilson3cce4692010-10-27 16:11:02 +0100469 intel_ring_advance(ring);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000470
Chris Wilson3cce4692010-10-27 16:11:02 +0100471 *result = seqno;
472 return 0;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700473}
474
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800475static u32
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000476ring_get_seqno(struct intel_ring_buffer *ring)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800477{
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000478 return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
479}
480
Chris Wilsonc6df5412010-12-15 09:56:50 +0000481static u32
482pc_render_get_seqno(struct intel_ring_buffer *ring)
483{
484 struct pipe_control *pc = ring->private;
485 return pc->cpu_page[0];
486}
487
Chris Wilson0f46832f2011-01-04 17:35:21 +0000488static void
489ironlake_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
490{
491 dev_priv->gt_irq_mask &= ~mask;
492 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
493 POSTING_READ(GTIMR);
494}
495
496static void
497ironlake_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
498{
499 dev_priv->gt_irq_mask |= mask;
500 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
501 POSTING_READ(GTIMR);
502}
503
504static void
505i915_enable_irq(drm_i915_private_t *dev_priv, u32 mask)
506{
507 dev_priv->irq_mask &= ~mask;
508 I915_WRITE(IMR, dev_priv->irq_mask);
509 POSTING_READ(IMR);
510}
511
512static void
513i915_disable_irq(drm_i915_private_t *dev_priv, u32 mask)
514{
515 dev_priv->irq_mask |= mask;
516 I915_WRITE(IMR, dev_priv->irq_mask);
517 POSTING_READ(IMR);
518}
519
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000520static bool
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000521render_ring_get_irq(struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700522{
Chris Wilson78501ea2010-10-27 12:18:21 +0100523 struct drm_device *dev = ring->dev;
Chris Wilson01a03332011-01-04 22:22:56 +0000524 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700525
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000526 if (!dev->irq_enabled)
527 return false;
528
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000529 spin_lock(&ring->irq_lock);
Chris Wilson01a03332011-01-04 22:22:56 +0000530 if (ring->irq_refcount++ == 0) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700531 if (HAS_PCH_SPLIT(dev))
Chris Wilson0f46832f2011-01-04 17:35:21 +0000532 ironlake_enable_irq(dev_priv,
533 GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700534 else
535 i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
536 }
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000537 spin_unlock(&ring->irq_lock);
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000538
539 return true;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700540}
541
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800542static void
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000543render_ring_put_irq(struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700544{
Chris Wilson78501ea2010-10-27 12:18:21 +0100545 struct drm_device *dev = ring->dev;
Chris Wilson01a03332011-01-04 22:22:56 +0000546 drm_i915_private_t *dev_priv = dev->dev_private;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700547
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000548 spin_lock(&ring->irq_lock);
Chris Wilson01a03332011-01-04 22:22:56 +0000549 if (--ring->irq_refcount == 0) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700550 if (HAS_PCH_SPLIT(dev))
Chris Wilson0f46832f2011-01-04 17:35:21 +0000551 ironlake_disable_irq(dev_priv,
552 GT_USER_INTERRUPT |
553 GT_PIPE_NOTIFY);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700554 else
555 i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
556 }
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000557 spin_unlock(&ring->irq_lock);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700558}
559
Chris Wilson78501ea2010-10-27 12:18:21 +0100560void intel_ring_setup_status_page(struct intel_ring_buffer *ring)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800561{
Chris Wilson78501ea2010-10-27 12:18:21 +0100562 drm_i915_private_t *dev_priv = ring->dev->dev_private;
563 u32 mmio = IS_GEN6(ring->dev) ?
564 RING_HWS_PGA_GEN6(ring->mmio_base) :
565 RING_HWS_PGA(ring->mmio_base);
566 I915_WRITE(mmio, (u32)ring->status_page.gfx_addr);
567 POSTING_READ(mmio);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800568}
569
Chris Wilsonb72f3ac2011-01-04 17:34:02 +0000570static int
Chris Wilson78501ea2010-10-27 12:18:21 +0100571bsd_ring_flush(struct intel_ring_buffer *ring,
572 u32 invalidate_domains,
573 u32 flush_domains)
Zou Nan haid1b851f2010-05-21 09:08:57 +0800574{
Chris Wilsonb72f3ac2011-01-04 17:34:02 +0000575 int ret;
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000576
Chris Wilsonb72f3ac2011-01-04 17:34:02 +0000577 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
578 return 0;
579
580 ret = intel_ring_begin(ring, 2);
581 if (ret)
582 return ret;
583
584 intel_ring_emit(ring, MI_FLUSH);
585 intel_ring_emit(ring, MI_NOOP);
586 intel_ring_advance(ring);
587 return 0;
Zou Nan haid1b851f2010-05-21 09:08:57 +0800588}
589
Chris Wilson3cce4692010-10-27 16:11:02 +0100590static int
Chris Wilson78501ea2010-10-27 12:18:21 +0100591ring_add_request(struct intel_ring_buffer *ring,
Chris Wilson3cce4692010-10-27 16:11:02 +0100592 u32 *result)
Zou Nan haid1b851f2010-05-21 09:08:57 +0800593{
594 u32 seqno;
Chris Wilson3cce4692010-10-27 16:11:02 +0100595 int ret;
596
597 ret = intel_ring_begin(ring, 4);
598 if (ret)
599 return ret;
Chris Wilson6f392d52010-08-07 11:01:22 +0100600
Chris Wilson78501ea2010-10-27 12:18:21 +0100601 seqno = i915_gem_get_seqno(ring->dev);
Chris Wilson6f392d52010-08-07 11:01:22 +0100602
Chris Wilson3cce4692010-10-27 16:11:02 +0100603 intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
604 intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
605 intel_ring_emit(ring, seqno);
606 intel_ring_emit(ring, MI_USER_INTERRUPT);
607 intel_ring_advance(ring);
Zou Nan haid1b851f2010-05-21 09:08:57 +0800608
609 DRM_DEBUG_DRIVER("%s %d\n", ring->name, seqno);
Chris Wilson3cce4692010-10-27 16:11:02 +0100610 *result = seqno;
611 return 0;
Zou Nan haid1b851f2010-05-21 09:08:57 +0800612}
613
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000614static bool
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000615ring_get_irq(struct intel_ring_buffer *ring, u32 flag)
Zou Nan haid1b851f2010-05-21 09:08:57 +0800616{
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000617 struct drm_device *dev = ring->dev;
Chris Wilson01a03332011-01-04 22:22:56 +0000618 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000619
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000620 if (!dev->irq_enabled)
621 return false;
622
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000623 spin_lock(&ring->irq_lock);
Chris Wilson01a03332011-01-04 22:22:56 +0000624 if (ring->irq_refcount++ == 0)
Chris Wilson0f46832f2011-01-04 17:35:21 +0000625 ironlake_enable_irq(dev_priv, flag);
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000626 spin_unlock(&ring->irq_lock);
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000627
628 return true;
Zou Nan haid1b851f2010-05-21 09:08:57 +0800629}
630
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000631static void
632ring_put_irq(struct intel_ring_buffer *ring, u32 flag)
Zou Nan haid1b851f2010-05-21 09:08:57 +0800633{
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000634 struct drm_device *dev = ring->dev;
Chris Wilson01a03332011-01-04 22:22:56 +0000635 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000636
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000637 spin_lock(&ring->irq_lock);
Chris Wilson01a03332011-01-04 22:22:56 +0000638 if (--ring->irq_refcount == 0)
Chris Wilson0f46832f2011-01-04 17:35:21 +0000639 ironlake_disable_irq(dev_priv, flag);
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000640 spin_unlock(&ring->irq_lock);
Chris Wilson0f46832f2011-01-04 17:35:21 +0000641}
642
643static bool
644gen6_ring_get_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
645{
646 struct drm_device *dev = ring->dev;
Chris Wilson01a03332011-01-04 22:22:56 +0000647 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson0f46832f2011-01-04 17:35:21 +0000648
649 if (!dev->irq_enabled)
650 return false;
651
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000652 spin_lock(&ring->irq_lock);
Chris Wilson01a03332011-01-04 22:22:56 +0000653 if (ring->irq_refcount++ == 0) {
Chris Wilson0f46832f2011-01-04 17:35:21 +0000654 ring->irq_mask &= ~rflag;
655 I915_WRITE_IMR(ring, ring->irq_mask);
656 ironlake_enable_irq(dev_priv, gflag);
Chris Wilson0f46832f2011-01-04 17:35:21 +0000657 }
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000658 spin_unlock(&ring->irq_lock);
Chris Wilson0f46832f2011-01-04 17:35:21 +0000659
660 return true;
661}
662
663static void
664gen6_ring_put_irq(struct intel_ring_buffer *ring, u32 gflag, u32 rflag)
665{
666 struct drm_device *dev = ring->dev;
Chris Wilson01a03332011-01-04 22:22:56 +0000667 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson0f46832f2011-01-04 17:35:21 +0000668
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000669 spin_lock(&ring->irq_lock);
Chris Wilson01a03332011-01-04 22:22:56 +0000670 if (--ring->irq_refcount == 0) {
Chris Wilson0f46832f2011-01-04 17:35:21 +0000671 ring->irq_mask |= rflag;
672 I915_WRITE_IMR(ring, ring->irq_mask);
673 ironlake_disable_irq(dev_priv, gflag);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000674 }
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000675 spin_unlock(&ring->irq_lock);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000676}
677
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000678static bool
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000679bsd_ring_get_irq(struct intel_ring_buffer *ring)
680{
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000681 return ring_get_irq(ring, GT_BSD_USER_INTERRUPT);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000682}
683static void
684bsd_ring_put_irq(struct intel_ring_buffer *ring)
685{
Chris Wilsonb13c2b92010-12-13 16:54:50 +0000686 ring_put_irq(ring, GT_BSD_USER_INTERRUPT);
Zou Nan haid1b851f2010-05-21 09:08:57 +0800687}
688
689static int
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000690ring_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
Zou Nan haid1b851f2010-05-21 09:08:57 +0800691{
Chris Wilsone1f99ce2010-10-27 12:45:26 +0100692 int ret;
Chris Wilson78501ea2010-10-27 12:18:21 +0100693
Chris Wilsone1f99ce2010-10-27 12:45:26 +0100694 ret = intel_ring_begin(ring, 2);
695 if (ret)
696 return ret;
697
Chris Wilson78501ea2010-10-27 12:18:21 +0100698 intel_ring_emit(ring,
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000699 MI_BATCH_BUFFER_START | (2 << 6) |
Chris Wilson78501ea2010-10-27 12:18:21 +0100700 MI_BATCH_NON_SECURE_I965);
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000701 intel_ring_emit(ring, offset);
Chris Wilson78501ea2010-10-27 12:18:21 +0100702 intel_ring_advance(ring);
703
Zou Nan haid1b851f2010-05-21 09:08:57 +0800704 return 0;
705}
706
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800707static int
Chris Wilson78501ea2010-10-27 12:18:21 +0100708render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000709 u32 offset, u32 len)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700710{
Chris Wilson78501ea2010-10-27 12:18:21 +0100711 struct drm_device *dev = ring->dev;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700712 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000713 int ret;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700714
Chris Wilson6f392d52010-08-07 11:01:22 +0100715 trace_i915_gem_request_submit(dev, dev_priv->next_seqno + 1);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700716
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000717 if (IS_I830(dev) || IS_845G(dev)) {
718 ret = intel_ring_begin(ring, 4);
719 if (ret)
720 return ret;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700721
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000722 intel_ring_emit(ring, MI_BATCH_BUFFER);
723 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
724 intel_ring_emit(ring, offset + len - 8);
725 intel_ring_emit(ring, 0);
726 } else {
727 ret = intel_ring_begin(ring, 2);
728 if (ret)
729 return ret;
Chris Wilsone1f99ce2010-10-27 12:45:26 +0100730
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000731 if (INTEL_INFO(dev)->gen >= 4) {
732 intel_ring_emit(ring,
733 MI_BATCH_BUFFER_START | (2 << 6) |
734 MI_BATCH_NON_SECURE_I965);
735 intel_ring_emit(ring, offset);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700736 } else {
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000737 intel_ring_emit(ring,
738 MI_BATCH_BUFFER_START | (2 << 6));
739 intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700740 }
741 }
Chris Wilsonc4e7a412010-11-30 14:10:25 +0000742 intel_ring_advance(ring);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700743
Eric Anholt62fdfea2010-05-21 13:26:39 -0700744 return 0;
745}
746
Chris Wilson78501ea2010-10-27 12:18:21 +0100747static void cleanup_status_page(struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700748{
Chris Wilson78501ea2010-10-27 12:18:21 +0100749 drm_i915_private_t *dev_priv = ring->dev->dev_private;
Chris Wilson05394f32010-11-08 19:18:58 +0000750 struct drm_i915_gem_object *obj;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700751
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800752 obj = ring->status_page.obj;
753 if (obj == NULL)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700754 return;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700755
Chris Wilson05394f32010-11-08 19:18:58 +0000756 kunmap(obj->pages[0]);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700757 i915_gem_object_unpin(obj);
Chris Wilson05394f32010-11-08 19:18:58 +0000758 drm_gem_object_unreference(&obj->base);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800759 ring->status_page.obj = NULL;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700760
761 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
Eric Anholt62fdfea2010-05-21 13:26:39 -0700762}
763
Chris Wilson78501ea2010-10-27 12:18:21 +0100764static int init_status_page(struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700765{
Chris Wilson78501ea2010-10-27 12:18:21 +0100766 struct drm_device *dev = ring->dev;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700767 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson05394f32010-11-08 19:18:58 +0000768 struct drm_i915_gem_object *obj;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700769 int ret;
770
Eric Anholt62fdfea2010-05-21 13:26:39 -0700771 obj = i915_gem_alloc_object(dev, 4096);
772 if (obj == NULL) {
773 DRM_ERROR("Failed to allocate status page\n");
774 ret = -ENOMEM;
775 goto err;
776 }
Chris Wilson05394f32010-11-08 19:18:58 +0000777 obj->agp_type = AGP_USER_CACHED_MEMORY;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700778
Daniel Vetter75e9e912010-11-04 17:11:09 +0100779 ret = i915_gem_object_pin(obj, 4096, true);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700780 if (ret != 0) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700781 goto err_unref;
782 }
783
Chris Wilson05394f32010-11-08 19:18:58 +0000784 ring->status_page.gfx_addr = obj->gtt_offset;
785 ring->status_page.page_addr = kmap(obj->pages[0]);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800786 if (ring->status_page.page_addr == NULL) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700787 memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
Eric Anholt62fdfea2010-05-21 13:26:39 -0700788 goto err_unpin;
789 }
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800790 ring->status_page.obj = obj;
791 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700792
Chris Wilson78501ea2010-10-27 12:18:21 +0100793 intel_ring_setup_status_page(ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800794 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
795 ring->name, ring->status_page.gfx_addr);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700796
797 return 0;
798
799err_unpin:
800 i915_gem_object_unpin(obj);
801err_unref:
Chris Wilson05394f32010-11-08 19:18:58 +0000802 drm_gem_object_unreference(&obj->base);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700803err:
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800804 return ret;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700805}
806
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800807int intel_init_ring_buffer(struct drm_device *dev,
Chris Wilsonab6f8e32010-09-19 17:53:44 +0100808 struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700809{
Chris Wilson05394f32010-11-08 19:18:58 +0000810 struct drm_i915_gem_object *obj;
Chris Wilsondd785e32010-08-07 11:01:34 +0100811 int ret;
812
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800813 ring->dev = dev;
Chris Wilson23bc5982010-09-29 16:10:57 +0100814 INIT_LIST_HEAD(&ring->active_list);
815 INIT_LIST_HEAD(&ring->request_list);
Chris Wilson64193402010-10-24 12:38:05 +0100816 INIT_LIST_HEAD(&ring->gpu_write_list);
Chris Wilson0dc79fb2011-01-05 10:32:24 +0000817
818 spin_lock_init(&ring->irq_lock);
Chris Wilson0f46832f2011-01-04 17:35:21 +0000819 ring->irq_mask = ~0;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700820
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800821 if (I915_NEED_GFX_HWS(dev)) {
Chris Wilson78501ea2010-10-27 12:18:21 +0100822 ret = init_status_page(ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800823 if (ret)
824 return ret;
825 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700826
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800827 obj = i915_gem_alloc_object(dev, ring->size);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700828 if (obj == NULL) {
829 DRM_ERROR("Failed to allocate ringbuffer\n");
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800830 ret = -ENOMEM;
Chris Wilsondd785e32010-08-07 11:01:34 +0100831 goto err_hws;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700832 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700833
Chris Wilson05394f32010-11-08 19:18:58 +0000834 ring->obj = obj;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800835
Daniel Vetter75e9e912010-11-04 17:11:09 +0100836 ret = i915_gem_object_pin(obj, PAGE_SIZE, true);
Chris Wilsondd785e32010-08-07 11:01:34 +0100837 if (ret)
838 goto err_unref;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700839
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800840 ring->map.size = ring->size;
Chris Wilson05394f32010-11-08 19:18:58 +0000841 ring->map.offset = dev->agp->base + obj->gtt_offset;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700842 ring->map.type = 0;
843 ring->map.flags = 0;
844 ring->map.mtrr = 0;
845
846 drm_core_ioremap_wc(&ring->map, dev);
847 if (ring->map.handle == NULL) {
848 DRM_ERROR("Failed to map ringbuffer.\n");
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800849 ret = -EINVAL;
Chris Wilsondd785e32010-08-07 11:01:34 +0100850 goto err_unpin;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700851 }
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800852
Eric Anholt62fdfea2010-05-21 13:26:39 -0700853 ring->virtual_start = ring->map.handle;
Chris Wilson78501ea2010-10-27 12:18:21 +0100854 ret = ring->init(ring);
Chris Wilsondd785e32010-08-07 11:01:34 +0100855 if (ret)
856 goto err_unmap;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700857
Chris Wilson55249ba2010-12-22 14:04:47 +0000858 /* Workaround an erratum on the i830 which causes a hang if
859 * the TAIL pointer points to within the last 2 cachelines
860 * of the buffer.
861 */
862 ring->effective_size = ring->size;
863 if (IS_I830(ring->dev))
864 ring->effective_size -= 128;
865
Chris Wilsonc584fe42010-10-29 18:15:52 +0100866 return 0;
Chris Wilsondd785e32010-08-07 11:01:34 +0100867
868err_unmap:
869 drm_core_ioremapfree(&ring->map, dev);
870err_unpin:
871 i915_gem_object_unpin(obj);
872err_unref:
Chris Wilson05394f32010-11-08 19:18:58 +0000873 drm_gem_object_unreference(&obj->base);
874 ring->obj = NULL;
Chris Wilsondd785e32010-08-07 11:01:34 +0100875err_hws:
Chris Wilson78501ea2010-10-27 12:18:21 +0100876 cleanup_status_page(ring);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800877 return ret;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700878}
879
Chris Wilson78501ea2010-10-27 12:18:21 +0100880void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700881{
Chris Wilson33626e62010-10-29 16:18:36 +0100882 struct drm_i915_private *dev_priv;
883 int ret;
884
Chris Wilson05394f32010-11-08 19:18:58 +0000885 if (ring->obj == NULL)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700886 return;
887
Chris Wilson33626e62010-10-29 16:18:36 +0100888 /* Disable the ring buffer. The ring must be idle at this point */
889 dev_priv = ring->dev->dev_private;
890 ret = intel_wait_ring_buffer(ring, ring->size - 8);
891 I915_WRITE_CTL(ring, 0);
892
Chris Wilson78501ea2010-10-27 12:18:21 +0100893 drm_core_ioremapfree(&ring->map, ring->dev);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700894
Chris Wilson05394f32010-11-08 19:18:58 +0000895 i915_gem_object_unpin(ring->obj);
896 drm_gem_object_unreference(&ring->obj->base);
897 ring->obj = NULL;
Chris Wilson78501ea2010-10-27 12:18:21 +0100898
Zou Nan hai8d192152010-11-02 16:31:01 +0800899 if (ring->cleanup)
900 ring->cleanup(ring);
901
Chris Wilson78501ea2010-10-27 12:18:21 +0100902 cleanup_status_page(ring);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700903}
904
Chris Wilson78501ea2010-10-27 12:18:21 +0100905static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700906{
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800907 unsigned int *virt;
Chris Wilson55249ba2010-12-22 14:04:47 +0000908 int rem = ring->size - ring->tail;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700909
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800910 if (ring->space < rem) {
Chris Wilson78501ea2010-10-27 12:18:21 +0100911 int ret = intel_wait_ring_buffer(ring, rem);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700912 if (ret)
913 return ret;
914 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700915
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800916 virt = (unsigned int *)(ring->virtual_start + ring->tail);
Chris Wilson1741dd42010-08-04 15:18:12 +0100917 rem /= 8;
918 while (rem--) {
Eric Anholt62fdfea2010-05-21 13:26:39 -0700919 *virt++ = MI_NOOP;
Chris Wilson1741dd42010-08-04 15:18:12 +0100920 *virt++ = MI_NOOP;
921 }
Eric Anholt62fdfea2010-05-21 13:26:39 -0700922
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800923 ring->tail = 0;
Chris Wilson43ed3402010-07-01 17:53:00 +0100924 ring->space = ring->head - 8;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700925
926 return 0;
927}
928
Chris Wilson78501ea2010-10-27 12:18:21 +0100929int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
Eric Anholt62fdfea2010-05-21 13:26:39 -0700930{
Chris Wilson78501ea2010-10-27 12:18:21 +0100931 struct drm_device *dev = ring->dev;
Zou Nan haicae58522010-11-09 17:17:32 +0800932 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson78501ea2010-10-27 12:18:21 +0100933 unsigned long end;
Chris Wilson6aa56062010-10-29 21:44:37 +0100934 u32 head;
935
Eric Anholt62fdfea2010-05-21 13:26:39 -0700936 trace_i915_ring_wait_begin (dev);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800937 end = jiffies + 3 * HZ;
938 do {
Chris Wilson8c0a6bf2010-12-09 12:56:37 +0000939 /* If the reported head position has wrapped or hasn't advanced,
940 * fallback to the slow and accurate path.
941 */
942 head = intel_read_status_page(ring, 4);
943 if (head < ring->actual_head)
944 head = I915_READ_HEAD(ring);
945 ring->actual_head = head;
946 ring->head = head & HEAD_ADDR;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700947 ring->space = ring->head - (ring->tail + 8);
948 if (ring->space < 0)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800949 ring->space += ring->size;
Eric Anholt62fdfea2010-05-21 13:26:39 -0700950 if (ring->space >= n) {
Chris Wilson78501ea2010-10-27 12:18:21 +0100951 trace_i915_ring_wait_end(dev);
Eric Anholt62fdfea2010-05-21 13:26:39 -0700952 return 0;
953 }
954
955 if (dev->primary->master) {
956 struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
957 if (master_priv->sarea_priv)
958 master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
959 }
Zou Nan haid1b851f2010-05-21 09:08:57 +0800960
Chris Wilsone60a0b12010-10-13 10:09:14 +0100961 msleep(1);
Chris Wilsonf4e0b292010-10-29 21:06:16 +0100962 if (atomic_read(&dev_priv->mm.wedged))
963 return -EAGAIN;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800964 } while (!time_after(jiffies, end));
Eric Anholt62fdfea2010-05-21 13:26:39 -0700965 trace_i915_ring_wait_end (dev);
966 return -EBUSY;
967}
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800968
Chris Wilsone1f99ce2010-10-27 12:45:26 +0100969int intel_ring_begin(struct intel_ring_buffer *ring,
970 int num_dwords)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800971{
Zou Nan haibe26a102010-06-12 17:40:24 +0800972 int n = 4*num_dwords;
Chris Wilsone1f99ce2010-10-27 12:45:26 +0100973 int ret;
Chris Wilson78501ea2010-10-27 12:18:21 +0100974
Chris Wilson55249ba2010-12-22 14:04:47 +0000975 if (unlikely(ring->tail + n > ring->effective_size)) {
Chris Wilsone1f99ce2010-10-27 12:45:26 +0100976 ret = intel_wrap_ring_buffer(ring);
977 if (unlikely(ret))
978 return ret;
979 }
Chris Wilson78501ea2010-10-27 12:18:21 +0100980
Chris Wilsone1f99ce2010-10-27 12:45:26 +0100981 if (unlikely(ring->space < n)) {
982 ret = intel_wait_ring_buffer(ring, n);
983 if (unlikely(ret))
984 return ret;
985 }
Chris Wilsond97ed332010-08-04 15:18:13 +0100986
987 ring->space -= n;
Chris Wilsone1f99ce2010-10-27 12:45:26 +0100988 return 0;
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800989}
990
Chris Wilson78501ea2010-10-27 12:18:21 +0100991void intel_ring_advance(struct intel_ring_buffer *ring)
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800992{
Chris Wilsond97ed332010-08-04 15:18:13 +0100993 ring->tail &= ring->size - 1;
Chris Wilson78501ea2010-10-27 12:18:21 +0100994 ring->write_tail(ring, ring->tail);
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800995}
996
Chris Wilsone0708682010-09-19 14:46:27 +0100997static const struct intel_ring_buffer render_ring = {
Zou Nan hai8187a2b2010-05-21 09:08:55 +0800998 .name = "render ring",
Chris Wilson92204342010-09-18 11:02:01 +0100999 .id = RING_RENDER,
Daniel Vetter333e9fe2010-08-02 16:24:01 +02001000 .mmio_base = RENDER_RING_BASE,
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001001 .size = 32 * PAGE_SIZE,
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001002 .init = init_render_ring,
Chris Wilson297b0c52010-10-22 17:02:41 +01001003 .write_tail = ring_write_tail,
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001004 .flush = render_ring_flush,
1005 .add_request = render_ring_add_request,
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001006 .get_seqno = ring_get_seqno,
1007 .irq_get = render_ring_get_irq,
1008 .irq_put = render_ring_put_irq,
Chris Wilson78501ea2010-10-27 12:18:21 +01001009 .dispatch_execbuffer = render_ring_dispatch_execbuffer,
Chris Wilsonc6df5412010-12-15 09:56:50 +00001010 .cleanup = render_ring_cleanup,
Zou Nan hai8187a2b2010-05-21 09:08:55 +08001011};
Zou Nan haid1b851f2010-05-21 09:08:57 +08001012
1013/* ring buffer for bit-stream decoder */
1014
Chris Wilsone0708682010-09-19 14:46:27 +01001015static const struct intel_ring_buffer bsd_ring = {
Zou Nan haid1b851f2010-05-21 09:08:57 +08001016 .name = "bsd ring",
Chris Wilson92204342010-09-18 11:02:01 +01001017 .id = RING_BSD,
Daniel Vetter333e9fe2010-08-02 16:24:01 +02001018 .mmio_base = BSD_RING_BASE,
Zou Nan haid1b851f2010-05-21 09:08:57 +08001019 .size = 32 * PAGE_SIZE,
Chris Wilson78501ea2010-10-27 12:18:21 +01001020 .init = init_ring_common,
Chris Wilson297b0c52010-10-22 17:02:41 +01001021 .write_tail = ring_write_tail,
Zou Nan haid1b851f2010-05-21 09:08:57 +08001022 .flush = bsd_ring_flush,
Chris Wilson549f7362010-10-19 11:19:32 +01001023 .add_request = ring_add_request,
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001024 .get_seqno = ring_get_seqno,
1025 .irq_get = bsd_ring_get_irq,
1026 .irq_put = bsd_ring_put_irq,
Chris Wilson78501ea2010-10-27 12:18:21 +01001027 .dispatch_execbuffer = ring_dispatch_execbuffer,
Zou Nan haid1b851f2010-05-21 09:08:57 +08001028};
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001029
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001030
Chris Wilson78501ea2010-10-27 12:18:21 +01001031static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
Chris Wilson297b0c52010-10-22 17:02:41 +01001032 u32 value)
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001033{
Chris Wilson78501ea2010-10-27 12:18:21 +01001034 drm_i915_private_t *dev_priv = ring->dev->dev_private;
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001035
1036 /* Every tail move must follow the sequence below */
1037 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1038 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1039 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_DISABLE);
1040 I915_WRITE(GEN6_BSD_RNCID, 0x0);
1041
1042 if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
1043 GEN6_BSD_SLEEP_PSMI_CONTROL_IDLE_INDICATOR) == 0,
1044 50))
1045 DRM_ERROR("timed out waiting for IDLE Indicator\n");
1046
Daniel Vetter870e86d2010-08-02 16:29:44 +02001047 I915_WRITE_TAIL(ring, value);
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001048 I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
1049 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_MODIFY_MASK |
1050 GEN6_BSD_SLEEP_PSMI_CONTROL_RC_ILDL_MESSAGE_ENABLE);
1051}
1052
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001053static int gen6_ring_flush(struct intel_ring_buffer *ring,
1054 u32 invalidate_domains,
1055 u32 flush_domains)
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001056{
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001057 int ret;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001058
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001059 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
1060 return 0;
1061
1062 ret = intel_ring_begin(ring, 4);
1063 if (ret)
1064 return ret;
1065
1066 intel_ring_emit(ring, MI_FLUSH_DW);
1067 intel_ring_emit(ring, 0);
1068 intel_ring_emit(ring, 0);
1069 intel_ring_emit(ring, 0);
1070 intel_ring_advance(ring);
1071 return 0;
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001072}
1073
1074static int
Chris Wilson78501ea2010-10-27 12:18:21 +01001075gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
Chris Wilsonc4e7a412010-11-30 14:10:25 +00001076 u32 offset, u32 len)
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001077{
Chris Wilsone1f99ce2010-10-27 12:45:26 +01001078 int ret;
Chris Wilsonab6f8e32010-09-19 17:53:44 +01001079
Chris Wilsone1f99ce2010-10-27 12:45:26 +01001080 ret = intel_ring_begin(ring, 2);
1081 if (ret)
1082 return ret;
1083
Chris Wilson78501ea2010-10-27 12:18:21 +01001084 intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_NON_SECURE_I965);
Chris Wilsonab6f8e32010-09-19 17:53:44 +01001085 /* bit0-7 is the length on GEN6+ */
Chris Wilsonc4e7a412010-11-30 14:10:25 +00001086 intel_ring_emit(ring, offset);
Chris Wilson78501ea2010-10-27 12:18:21 +01001087 intel_ring_advance(ring);
Chris Wilsonab6f8e32010-09-19 17:53:44 +01001088
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001089 return 0;
1090}
1091
Chris Wilsonb13c2b92010-12-13 16:54:50 +00001092static bool
Chris Wilson0f46832f2011-01-04 17:35:21 +00001093gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
1094{
1095 return gen6_ring_get_irq(ring,
1096 GT_USER_INTERRUPT,
1097 GEN6_RENDER_USER_INTERRUPT);
1098}
1099
1100static void
1101gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
1102{
1103 return gen6_ring_put_irq(ring,
1104 GT_USER_INTERRUPT,
1105 GEN6_RENDER_USER_INTERRUPT);
1106}
1107
1108static bool
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001109gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
1110{
Chris Wilson0f46832f2011-01-04 17:35:21 +00001111 return gen6_ring_get_irq(ring,
1112 GT_GEN6_BSD_USER_INTERRUPT,
1113 GEN6_BSD_USER_INTERRUPT);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001114}
1115
1116static void
1117gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
1118{
Chris Wilson0f46832f2011-01-04 17:35:21 +00001119 return gen6_ring_put_irq(ring,
1120 GT_GEN6_BSD_USER_INTERRUPT,
1121 GEN6_BSD_USER_INTERRUPT);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001122}
1123
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001124/* ring buffer for Video Codec for Gen6+ */
Chris Wilsone0708682010-09-19 14:46:27 +01001125static const struct intel_ring_buffer gen6_bsd_ring = {
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001126 .name = "gen6 bsd ring",
1127 .id = RING_BSD,
1128 .mmio_base = GEN6_BSD_RING_BASE,
1129 .size = 32 * PAGE_SIZE,
1130 .init = init_ring_common,
1131 .write_tail = gen6_bsd_ring_write_tail,
1132 .flush = gen6_ring_flush,
1133 .add_request = gen6_add_request,
1134 .get_seqno = ring_get_seqno,
1135 .irq_get = gen6_bsd_ring_get_irq,
1136 .irq_put = gen6_bsd_ring_put_irq,
1137 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
Chris Wilson549f7362010-10-19 11:19:32 +01001138};
1139
1140/* Blitter support (SandyBridge+) */
1141
Chris Wilsonb13c2b92010-12-13 16:54:50 +00001142static bool
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001143blt_ring_get_irq(struct intel_ring_buffer *ring)
Chris Wilson549f7362010-10-19 11:19:32 +01001144{
Chris Wilson0f46832f2011-01-04 17:35:21 +00001145 return gen6_ring_get_irq(ring,
1146 GT_BLT_USER_INTERRUPT,
1147 GEN6_BLITTER_USER_INTERRUPT);
Chris Wilson549f7362010-10-19 11:19:32 +01001148}
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001149
Chris Wilson549f7362010-10-19 11:19:32 +01001150static void
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001151blt_ring_put_irq(struct intel_ring_buffer *ring)
Chris Wilson549f7362010-10-19 11:19:32 +01001152{
Chris Wilson0f46832f2011-01-04 17:35:21 +00001153 gen6_ring_put_irq(ring,
1154 GT_BLT_USER_INTERRUPT,
1155 GEN6_BLITTER_USER_INTERRUPT);
Chris Wilson549f7362010-10-19 11:19:32 +01001156}
1157
Zou Nan hai8d192152010-11-02 16:31:01 +08001158
1159/* Workaround for some stepping of SNB,
1160 * each time when BLT engine ring tail moved,
1161 * the first command in the ring to be parsed
1162 * should be MI_BATCH_BUFFER_START
1163 */
1164#define NEED_BLT_WORKAROUND(dev) \
1165 (IS_GEN6(dev) && (dev->pdev->revision < 8))
1166
1167static inline struct drm_i915_gem_object *
1168to_blt_workaround(struct intel_ring_buffer *ring)
1169{
1170 return ring->private;
1171}
1172
1173static int blt_ring_init(struct intel_ring_buffer *ring)
1174{
1175 if (NEED_BLT_WORKAROUND(ring->dev)) {
1176 struct drm_i915_gem_object *obj;
Chris Wilson27153f72010-11-02 11:17:23 +00001177 u32 *ptr;
Zou Nan hai8d192152010-11-02 16:31:01 +08001178 int ret;
1179
Chris Wilson05394f32010-11-08 19:18:58 +00001180 obj = i915_gem_alloc_object(ring->dev, 4096);
Zou Nan hai8d192152010-11-02 16:31:01 +08001181 if (obj == NULL)
1182 return -ENOMEM;
1183
Chris Wilson05394f32010-11-08 19:18:58 +00001184 ret = i915_gem_object_pin(obj, 4096, true);
Zou Nan hai8d192152010-11-02 16:31:01 +08001185 if (ret) {
1186 drm_gem_object_unreference(&obj->base);
1187 return ret;
1188 }
1189
1190 ptr = kmap(obj->pages[0]);
Chris Wilson27153f72010-11-02 11:17:23 +00001191 *ptr++ = MI_BATCH_BUFFER_END;
1192 *ptr++ = MI_NOOP;
Zou Nan hai8d192152010-11-02 16:31:01 +08001193 kunmap(obj->pages[0]);
1194
Chris Wilson05394f32010-11-08 19:18:58 +00001195 ret = i915_gem_object_set_to_gtt_domain(obj, false);
Zou Nan hai8d192152010-11-02 16:31:01 +08001196 if (ret) {
Chris Wilson05394f32010-11-08 19:18:58 +00001197 i915_gem_object_unpin(obj);
Zou Nan hai8d192152010-11-02 16:31:01 +08001198 drm_gem_object_unreference(&obj->base);
1199 return ret;
1200 }
1201
1202 ring->private = obj;
1203 }
1204
1205 return init_ring_common(ring);
1206}
1207
1208static int blt_ring_begin(struct intel_ring_buffer *ring,
1209 int num_dwords)
1210{
1211 if (ring->private) {
1212 int ret = intel_ring_begin(ring, num_dwords+2);
1213 if (ret)
1214 return ret;
1215
1216 intel_ring_emit(ring, MI_BATCH_BUFFER_START);
1217 intel_ring_emit(ring, to_blt_workaround(ring)->gtt_offset);
1218
1219 return 0;
1220 } else
1221 return intel_ring_begin(ring, 4);
1222}
1223
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001224static int blt_ring_flush(struct intel_ring_buffer *ring,
Zou Nan hai8d192152010-11-02 16:31:01 +08001225 u32 invalidate_domains,
1226 u32 flush_domains)
1227{
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001228 int ret;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001229
Chris Wilsonb72f3ac2011-01-04 17:34:02 +00001230 if ((flush_domains & I915_GEM_DOMAIN_RENDER) == 0)
1231 return 0;
1232
1233 ret = blt_ring_begin(ring, 4);
1234 if (ret)
1235 return ret;
1236
1237 intel_ring_emit(ring, MI_FLUSH_DW);
1238 intel_ring_emit(ring, 0);
1239 intel_ring_emit(ring, 0);
1240 intel_ring_emit(ring, 0);
1241 intel_ring_advance(ring);
1242 return 0;
Zou Nan hai8d192152010-11-02 16:31:01 +08001243}
1244
Zou Nan hai8d192152010-11-02 16:31:01 +08001245static void blt_ring_cleanup(struct intel_ring_buffer *ring)
1246{
1247 if (!ring->private)
1248 return;
1249
1250 i915_gem_object_unpin(ring->private);
1251 drm_gem_object_unreference(ring->private);
1252 ring->private = NULL;
1253}
1254
Chris Wilson549f7362010-10-19 11:19:32 +01001255static const struct intel_ring_buffer gen6_blt_ring = {
1256 .name = "blt ring",
1257 .id = RING_BLT,
1258 .mmio_base = BLT_RING_BASE,
1259 .size = 32 * PAGE_SIZE,
Zou Nan hai8d192152010-11-02 16:31:01 +08001260 .init = blt_ring_init,
Chris Wilson297b0c52010-10-22 17:02:41 +01001261 .write_tail = ring_write_tail,
Zou Nan hai8d192152010-11-02 16:31:01 +08001262 .flush = blt_ring_flush,
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001263 .add_request = gen6_add_request,
1264 .get_seqno = ring_get_seqno,
1265 .irq_get = blt_ring_get_irq,
1266 .irq_put = blt_ring_put_irq,
Chris Wilson78501ea2010-10-27 12:18:21 +01001267 .dispatch_execbuffer = gen6_ring_dispatch_execbuffer,
Zou Nan hai8d192152010-11-02 16:31:01 +08001268 .cleanup = blt_ring_cleanup,
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001269};
1270
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001271int intel_init_render_ring_buffer(struct drm_device *dev)
1272{
1273 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001274 struct intel_ring_buffer *ring = &dev_priv->ring[RCS];
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001275
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001276 *ring = render_ring;
1277 if (INTEL_INFO(dev)->gen >= 6) {
1278 ring->add_request = gen6_add_request;
Chris Wilson0f46832f2011-01-04 17:35:21 +00001279 ring->irq_get = gen6_render_ring_get_irq;
1280 ring->irq_put = gen6_render_ring_put_irq;
Chris Wilsonc6df5412010-12-15 09:56:50 +00001281 } else if (IS_GEN5(dev)) {
1282 ring->add_request = pc_render_add_request;
1283 ring->get_seqno = pc_render_get_seqno;
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001284 }
1285
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001286 if (!I915_NEED_GFX_HWS(dev)) {
1287 ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
1288 memset(ring->status_page.page_addr, 0, PAGE_SIZE);
1289 }
1290
1291 return intel_init_ring_buffer(dev, ring);
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001292}
1293
1294int intel_init_bsd_ring_buffer(struct drm_device *dev)
1295{
1296 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001297 struct intel_ring_buffer *ring = &dev_priv->ring[VCS];
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001298
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001299 if (IS_GEN6(dev))
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001300 *ring = gen6_bsd_ring;
Xiang, Haihao881f47b2010-09-19 14:40:43 +01001301 else
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001302 *ring = bsd_ring;
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001303
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001304 return intel_init_ring_buffer(dev, ring);
Xiang, Haihao5c1143b2010-09-16 10:43:11 +08001305}
Chris Wilson549f7362010-10-19 11:19:32 +01001306
1307int intel_init_blt_ring_buffer(struct drm_device *dev)
1308{
1309 drm_i915_private_t *dev_priv = dev->dev_private;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001310 struct intel_ring_buffer *ring = &dev_priv->ring[BCS];
Chris Wilson549f7362010-10-19 11:19:32 +01001311
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001312 *ring = gen6_blt_ring;
Chris Wilson549f7362010-10-19 11:19:32 +01001313
Chris Wilson1ec14ad2010-12-04 11:30:53 +00001314 return intel_init_ring_buffer(dev, ring);
Chris Wilson549f7362010-10-19 11:19:32 +01001315}