blob: a42b738e79e77db02960a8ce3cd8fbadaf21a3a4 [file] [log] [blame]
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +01001/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
Chris Wilsonf636edb2017-10-09 12:02:57 +010025#include <drm/drm_print.h>
26
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +010027#include "i915_drv.h"
Weinan Li1fd51d92017-10-15 11:55:25 +080028#include "i915_vgpu.h"
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +010029#include "intel_ringbuffer.h"
30#include "intel_lrc.h"
31
Joonas Lahtinen63ffbcd2017-04-28 10:53:36 +030032/* Haswell does have the CXT_SIZE register however it does not appear to be
33 * valid. Now, docs explain in dwords what is in the context object. The full
34 * size is 70720 bytes, however, the power context and execlist context will
35 * never be saved (power context is stored elsewhere, and execlists don't work
36 * on HSW) - so the final size, including the extra state required for the
37 * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
38 */
39#define HSW_CXT_TOTAL_SIZE (17 * PAGE_SIZE)
40/* Same as Haswell, but 72064 bytes now. */
41#define GEN8_CXT_TOTAL_SIZE (18 * PAGE_SIZE)
42
43#define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
44#define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
Oscar Mateo3cf19342017-10-04 08:39:52 -070045#define GEN10_LR_CONTEXT_RENDER_SIZE (18 * PAGE_SIZE)
Joonas Lahtinen63ffbcd2017-04-28 10:53:36 +030046
47#define GEN8_LR_CONTEXT_OTHER_SIZE ( 2 * PAGE_SIZE)
48
Oscar Mateob8400f02017-04-10 07:34:32 -070049struct engine_class_info {
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +010050 const char *name;
Oscar Mateob8400f02017-04-10 07:34:32 -070051 int (*init_legacy)(struct intel_engine_cs *engine);
52 int (*init_execlists)(struct intel_engine_cs *engine);
Tvrtko Ursulin1803fcbc2017-11-10 14:26:27 +000053
54 u8 uabi_class;
Oscar Mateob8400f02017-04-10 07:34:32 -070055};
56
57static const struct engine_class_info intel_engine_classes[] = {
58 [RENDER_CLASS] = {
59 .name = "rcs",
60 .init_execlists = logical_render_ring_init,
61 .init_legacy = intel_init_render_ring_buffer,
Tvrtko Ursulin1803fcbc2017-11-10 14:26:27 +000062 .uabi_class = I915_ENGINE_CLASS_RENDER,
Oscar Mateob8400f02017-04-10 07:34:32 -070063 },
64 [COPY_ENGINE_CLASS] = {
65 .name = "bcs",
66 .init_execlists = logical_xcs_ring_init,
67 .init_legacy = intel_init_blt_ring_buffer,
Tvrtko Ursulin1803fcbc2017-11-10 14:26:27 +000068 .uabi_class = I915_ENGINE_CLASS_COPY,
Oscar Mateob8400f02017-04-10 07:34:32 -070069 },
70 [VIDEO_DECODE_CLASS] = {
71 .name = "vcs",
72 .init_execlists = logical_xcs_ring_init,
73 .init_legacy = intel_init_bsd_ring_buffer,
Tvrtko Ursulin1803fcbc2017-11-10 14:26:27 +000074 .uabi_class = I915_ENGINE_CLASS_VIDEO,
Oscar Mateob8400f02017-04-10 07:34:32 -070075 },
76 [VIDEO_ENHANCEMENT_CLASS] = {
77 .name = "vecs",
78 .init_execlists = logical_xcs_ring_init,
79 .init_legacy = intel_init_vebox_ring_buffer,
Tvrtko Ursulin1803fcbc2017-11-10 14:26:27 +000080 .uabi_class = I915_ENGINE_CLASS_VIDEO_ENHANCE,
Oscar Mateob8400f02017-04-10 07:34:32 -070081 },
82};
83
84struct engine_info {
Michal Wajdeczko237ae7c2017-03-01 20:26:15 +000085 unsigned int hw_id;
Chris Wilson1d39f282017-04-11 13:43:06 +010086 unsigned int uabi_id;
Daniele Ceraolo Spurio09081802017-04-10 07:34:29 -070087 u8 class;
88 u8 instance;
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +010089 u32 mmio_base;
90 unsigned irq_shift;
Oscar Mateob8400f02017-04-10 07:34:32 -070091};
92
93static const struct engine_info intel_engines[] = {
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +010094 [RCS] = {
Tvrtko Ursulin5ec2cf72016-08-16 17:04:20 +010095 .hw_id = RCS_HW,
Chris Wilson1d39f282017-04-11 13:43:06 +010096 .uabi_id = I915_EXEC_RENDER,
Daniele Ceraolo Spurio09081802017-04-10 07:34:29 -070097 .class = RENDER_CLASS,
98 .instance = 0,
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +010099 .mmio_base = RENDER_RING_BASE,
100 .irq_shift = GEN8_RCS_IRQ_SHIFT,
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100101 },
102 [BCS] = {
Tvrtko Ursulin5ec2cf72016-08-16 17:04:20 +0100103 .hw_id = BCS_HW,
Chris Wilson1d39f282017-04-11 13:43:06 +0100104 .uabi_id = I915_EXEC_BLT,
Daniele Ceraolo Spurio09081802017-04-10 07:34:29 -0700105 .class = COPY_ENGINE_CLASS,
106 .instance = 0,
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100107 .mmio_base = BLT_RING_BASE,
108 .irq_shift = GEN8_BCS_IRQ_SHIFT,
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100109 },
110 [VCS] = {
Tvrtko Ursulin5ec2cf72016-08-16 17:04:20 +0100111 .hw_id = VCS_HW,
Chris Wilson1d39f282017-04-11 13:43:06 +0100112 .uabi_id = I915_EXEC_BSD,
Daniele Ceraolo Spurio09081802017-04-10 07:34:29 -0700113 .class = VIDEO_DECODE_CLASS,
114 .instance = 0,
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100115 .mmio_base = GEN6_BSD_RING_BASE,
116 .irq_shift = GEN8_VCS1_IRQ_SHIFT,
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100117 },
118 [VCS2] = {
Tvrtko Ursulin5ec2cf72016-08-16 17:04:20 +0100119 .hw_id = VCS2_HW,
Chris Wilson1d39f282017-04-11 13:43:06 +0100120 .uabi_id = I915_EXEC_BSD,
Daniele Ceraolo Spurio09081802017-04-10 07:34:29 -0700121 .class = VIDEO_DECODE_CLASS,
122 .instance = 1,
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100123 .mmio_base = GEN8_BSD2_RING_BASE,
124 .irq_shift = GEN8_VCS2_IRQ_SHIFT,
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100125 },
126 [VECS] = {
Tvrtko Ursulin5ec2cf72016-08-16 17:04:20 +0100127 .hw_id = VECS_HW,
Chris Wilson1d39f282017-04-11 13:43:06 +0100128 .uabi_id = I915_EXEC_VEBOX,
Daniele Ceraolo Spurio09081802017-04-10 07:34:29 -0700129 .class = VIDEO_ENHANCEMENT_CLASS,
130 .instance = 0,
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100131 .mmio_base = VEBOX_RING_BASE,
132 .irq_shift = GEN8_VECS_IRQ_SHIFT,
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100133 },
134};
135
Joonas Lahtinen63ffbcd2017-04-28 10:53:36 +0300136/**
137 * ___intel_engine_context_size() - return the size of the context for an engine
138 * @dev_priv: i915 device private
139 * @class: engine class
140 *
141 * Each engine class may require a different amount of space for a context
142 * image.
143 *
144 * Return: size (in bytes) of an engine class specific context image
145 *
146 * Note: this size includes the HWSP, which is part of the context image
147 * in LRC mode, but does not include the "shared data page" used with
148 * GuC submission. The caller should account for this if using the GuC.
149 */
150static u32
151__intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
152{
153 u32 cxt_size;
154
155 BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
156
157 switch (class) {
158 case RENDER_CLASS:
159 switch (INTEL_GEN(dev_priv)) {
160 default:
161 MISSING_CASE(INTEL_GEN(dev_priv));
Rodrigo Vivif65f8412017-07-06 14:06:24 -0700162 case 10:
Oscar Mateo7fd0b1a2017-09-21 16:19:49 -0700163 return GEN10_LR_CONTEXT_RENDER_SIZE;
Joonas Lahtinen63ffbcd2017-04-28 10:53:36 +0300164 case 9:
165 return GEN9_LR_CONTEXT_RENDER_SIZE;
166 case 8:
Michal Wajdeczko4f044a82017-09-19 19:38:44 +0000167 return i915_modparams.enable_execlists ?
Joonas Lahtinen63ffbcd2017-04-28 10:53:36 +0300168 GEN8_LR_CONTEXT_RENDER_SIZE :
169 GEN8_CXT_TOTAL_SIZE;
170 case 7:
171 if (IS_HASWELL(dev_priv))
172 return HSW_CXT_TOTAL_SIZE;
173
174 cxt_size = I915_READ(GEN7_CXT_SIZE);
175 return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
176 PAGE_SIZE);
177 case 6:
178 cxt_size = I915_READ(CXT_SIZE);
179 return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
180 PAGE_SIZE);
181 case 5:
182 case 4:
183 case 3:
184 case 2:
185 /* For the special day when i810 gets merged. */
186 case 1:
187 return 0;
188 }
189 break;
190 default:
191 MISSING_CASE(class);
192 case VIDEO_DECODE_CLASS:
193 case VIDEO_ENHANCEMENT_CLASS:
194 case COPY_ENGINE_CLASS:
195 if (INTEL_GEN(dev_priv) < 8)
196 return 0;
197 return GEN8_LR_CONTEXT_OTHER_SIZE;
198 }
199}
200
Akash Goel3b3f1652016-10-13 22:44:48 +0530201static int
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100202intel_engine_setup(struct drm_i915_private *dev_priv,
203 enum intel_engine_id id)
204{
205 const struct engine_info *info = &intel_engines[id];
Oscar Mateob8400f02017-04-10 07:34:32 -0700206 const struct engine_class_info *class_info;
Akash Goel3b3f1652016-10-13 22:44:48 +0530207 struct intel_engine_cs *engine;
208
Oscar Mateob8400f02017-04-10 07:34:32 -0700209 GEM_BUG_ON(info->class >= ARRAY_SIZE(intel_engine_classes));
210 class_info = &intel_engine_classes[info->class];
211
Akash Goel3b3f1652016-10-13 22:44:48 +0530212 GEM_BUG_ON(dev_priv->engine[id]);
213 engine = kzalloc(sizeof(*engine), GFP_KERNEL);
214 if (!engine)
215 return -ENOMEM;
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100216
217 engine->id = id;
218 engine->i915 = dev_priv;
Oscar Mateo6e516142017-04-10 07:34:31 -0700219 WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s%u",
Oscar Mateob8400f02017-04-10 07:34:32 -0700220 class_info->name, info->instance) >=
221 sizeof(engine->name));
Tvrtko Ursulin5ec2cf72016-08-16 17:04:20 +0100222 engine->hw_id = engine->guc_id = info->hw_id;
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100223 engine->mmio_base = info->mmio_base;
224 engine->irq_shift = info->irq_shift;
Daniele Ceraolo Spurio09081802017-04-10 07:34:29 -0700225 engine->class = info->class;
226 engine->instance = info->instance;
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100227
Tvrtko Ursulin1803fcbc2017-11-10 14:26:27 +0000228 engine->uabi_id = info->uabi_id;
229 engine->uabi_class = class_info->uabi_class;
230
Joonas Lahtinen63ffbcd2017-04-28 10:53:36 +0300231 engine->context_size = __intel_engine_context_size(dev_priv,
232 engine->class);
233 if (WARN_ON(engine->context_size > BIT(20)))
234 engine->context_size = 0;
235
Chris Wilson0de91362016-11-14 20:41:01 +0000236 /* Nothing to do here, execute in order of dependencies */
237 engine->schedule = NULL;
238
Changbin Du3fc03062017-03-13 10:47:11 +0800239 ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
240
Akash Goel3b3f1652016-10-13 22:44:48 +0530241 dev_priv->engine[id] = engine;
242 return 0;
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100243}
244
245/**
Joonas Lahtinen63ffbcd2017-04-28 10:53:36 +0300246 * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
Tvrtko Ursulinbf9e8422016-12-01 14:16:38 +0000247 * @dev_priv: i915 device private
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100248 *
249 * Return: non-zero if the initialization failed.
250 */
Joonas Lahtinen63ffbcd2017-04-28 10:53:36 +0300251int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100252{
Tvrtko Ursulinc1bb1142016-08-10 16:22:10 +0100253 struct intel_device_info *device_info = mkwrite_device_info(dev_priv);
Chris Wilson5f9be052017-04-11 17:56:58 +0100254 const unsigned int ring_mask = INTEL_INFO(dev_priv)->ring_mask;
Akash Goel3b3f1652016-10-13 22:44:48 +0530255 struct intel_engine_cs *engine;
256 enum intel_engine_id id;
Chris Wilson5f9be052017-04-11 17:56:58 +0100257 unsigned int mask = 0;
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100258 unsigned int i;
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000259 int err;
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100260
Tvrtko Ursulin70006ad2016-10-13 11:02:56 +0100261 WARN_ON(ring_mask == 0);
262 WARN_ON(ring_mask &
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100263 GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));
264
265 for (i = 0; i < ARRAY_SIZE(intel_engines); i++) {
266 if (!HAS_ENGINE(dev_priv, i))
267 continue;
268
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000269 err = intel_engine_setup(dev_priv, i);
270 if (err)
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100271 goto cleanup;
272
273 mask |= ENGINE_MASK(i);
274 }
275
276 /*
277 * Catch failures to update intel_engines table when the new engines
278 * are added to the driver by a warning and disabling the forgotten
279 * engines.
280 */
Tvrtko Ursulin70006ad2016-10-13 11:02:56 +0100281 if (WARN_ON(mask != ring_mask))
Tvrtko Ursulinc1bb1142016-08-10 16:22:10 +0100282 device_info->ring_mask = mask;
283
Chris Wilson5f9be052017-04-11 17:56:58 +0100284 /* We always presume we have at least RCS available for later probing */
285 if (WARN_ON(!HAS_ENGINE(dev_priv, RCS))) {
286 err = -ENODEV;
287 goto cleanup;
288 }
289
Tvrtko Ursulinc1bb1142016-08-10 16:22:10 +0100290 device_info->num_rings = hweight32(mask);
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100291
Michel Thierryce453b32017-11-10 16:44:47 -0800292 i915_check_and_clear_faults(dev_priv);
293
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100294 return 0;
295
296cleanup:
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000297 for_each_engine(engine, dev_priv, id)
298 kfree(engine);
299 return err;
300}
301
302/**
Joonas Lahtinen63ffbcd2017-04-28 10:53:36 +0300303 * intel_engines_init() - init the Engine Command Streamers
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000304 * @dev_priv: i915 device private
305 *
306 * Return: non-zero if the initialization failed.
307 */
308int intel_engines_init(struct drm_i915_private *dev_priv)
309{
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000310 struct intel_engine_cs *engine;
311 enum intel_engine_id id, err_id;
Tvrtko Ursulin33def1f2017-06-16 14:03:38 +0100312 int err;
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000313
Akash Goel3b3f1652016-10-13 22:44:48 +0530314 for_each_engine(engine, dev_priv, id) {
Oscar Mateob8400f02017-04-10 07:34:32 -0700315 const struct engine_class_info *class_info =
316 &intel_engine_classes[engine->class];
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000317 int (*init)(struct intel_engine_cs *engine);
318
Michal Wajdeczko4f044a82017-09-19 19:38:44 +0000319 if (i915_modparams.enable_execlists)
Oscar Mateob8400f02017-04-10 07:34:32 -0700320 init = class_info->init_execlists;
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000321 else
Oscar Mateob8400f02017-04-10 07:34:32 -0700322 init = class_info->init_legacy;
Tvrtko Ursulin33def1f2017-06-16 14:03:38 +0100323
324 err = -EINVAL;
325 err_id = id;
326
327 if (GEM_WARN_ON(!init))
328 goto cleanup;
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000329
330 err = init(engine);
Tvrtko Ursulin33def1f2017-06-16 14:03:38 +0100331 if (err)
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000332 goto cleanup;
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000333
Chris Wilsonff44ad52017-03-16 17:13:03 +0000334 GEM_BUG_ON(!engine->submit_request);
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000335 }
336
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000337 return 0;
338
339cleanup:
340 for_each_engine(engine, dev_priv, id) {
Tvrtko Ursulin33def1f2017-06-16 14:03:38 +0100341 if (id >= err_id) {
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000342 kfree(engine);
Tvrtko Ursulin33def1f2017-06-16 14:03:38 +0100343 dev_priv->engine[id] = NULL;
344 } else {
Tvrtko Ursulin8ee7c6e2017-02-16 12:23:22 +0000345 dev_priv->gt.cleanup_engine(engine);
Tvrtko Ursulin33def1f2017-06-16 14:03:38 +0100346 }
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100347 }
Chris Wilsonbb8f0f52017-01-24 11:01:34 +0000348 return err;
Tvrtko Ursulin88d2ba22016-07-13 16:03:40 +0100349}
350
Chris Wilson73cb9702016-10-28 13:58:46 +0100351void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno)
Chris Wilson57f275a2016-08-15 10:49:00 +0100352{
353 struct drm_i915_private *dev_priv = engine->i915;
354
355 /* Our semaphore implementation is strictly monotonic (i.e. we proceed
356 * so long as the semaphore value in the register/page is greater
357 * than the sync value), so whenever we reset the seqno,
358 * so long as we reset the tracking semaphore value to 0, it will
359 * always be before the next request's seqno. If we don't reset
360 * the semaphore value, then when the seqno moves backwards all
361 * future waits will complete instantly (causing rendering corruption).
362 */
363 if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
364 I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
365 I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
366 if (HAS_VEBOX(dev_priv))
367 I915_WRITE(RING_SYNC_2(engine->mmio_base), 0);
368 }
Chris Wilson51d545d2016-08-15 10:49:02 +0100369 if (dev_priv->semaphore) {
370 struct page *page = i915_vma_first_page(dev_priv->semaphore);
371 void *semaphores;
372
373 /* Semaphores are in noncoherent memory, flush to be safe */
Chris Wilson24caf652017-03-20 14:56:09 +0000374 semaphores = kmap_atomic(page);
Chris Wilson57f275a2016-08-15 10:49:00 +0100375 memset(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
376 0, I915_NUM_ENGINES * gen8_semaphore_seqno_size);
Chris Wilson51d545d2016-08-15 10:49:02 +0100377 drm_clflush_virt_range(semaphores + GEN8_SEMAPHORE_OFFSET(engine->id, 0),
378 I915_NUM_ENGINES * gen8_semaphore_seqno_size);
Chris Wilson24caf652017-03-20 14:56:09 +0000379 kunmap_atomic(semaphores);
Chris Wilson57f275a2016-08-15 10:49:00 +0100380 }
Chris Wilson57f275a2016-08-15 10:49:00 +0100381
382 intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
Chris Wilson14a6bbf2017-03-14 11:14:52 +0000383 clear_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
Chris Wilson73cb9702016-10-28 13:58:46 +0100384
Chris Wilson57f275a2016-08-15 10:49:00 +0100385 /* After manually advancing the seqno, fake the interrupt in case
386 * there are any waiters for that seqno.
387 */
388 intel_engine_wakeup(engine);
Chris Wilson2ca9faa2017-04-05 16:30:54 +0100389
390 GEM_BUG_ON(intel_engine_get_seqno(engine) != seqno);
Chris Wilson57f275a2016-08-15 10:49:00 +0100391}
392
Chris Wilson73cb9702016-10-28 13:58:46 +0100393static void intel_engine_init_timeline(struct intel_engine_cs *engine)
Chris Wilsondcff85c2016-08-05 10:14:11 +0100394{
Chris Wilson73cb9702016-10-28 13:58:46 +0100395 engine->timeline = &engine->i915->gt.global_timeline.engine[engine->id];
Chris Wilsondcff85c2016-08-05 10:14:11 +0100396}
397
Mika Kuoppala19df9a52017-09-22 15:43:04 +0300398static bool csb_force_mmio(struct drm_i915_private *i915)
399{
Mika Kuoppala19df9a52017-09-22 15:43:04 +0300400 /*
401 * IOMMU adds unpredictable latency causing the CSB write (from the
402 * GPU into the HWSP) to only be visible some time after the interrupt
403 * (missed breadcrumb syndrome).
404 */
405 if (intel_vtd_active())
406 return true;
407
Weinan Li1fd51d92017-10-15 11:55:25 +0800408 /* Older GVT emulation depends upon intercepting CSB mmio */
409 if (intel_vgpu_active(i915) && !intel_vgpu_has_hwsp_emulation(i915))
410 return true;
411
Mika Kuoppala19df9a52017-09-22 15:43:04 +0300412 return false;
413}
414
415static void intel_engine_init_execlist(struct intel_engine_cs *engine)
416{
417 struct intel_engine_execlists * const execlists = &engine->execlists;
418
419 execlists->csb_use_mmio = csb_force_mmio(engine->i915);
420
Mika Kuoppala76e70082017-09-22 15:43:07 +0300421 execlists->port_mask = 1;
422 BUILD_BUG_ON_NOT_POWER_OF_2(execlists_num_ports(execlists));
423 GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
424
Mika Kuoppala19df9a52017-09-22 15:43:04 +0300425 execlists->queue = RB_ROOT;
426 execlists->first = NULL;
427}
428
Tvrtko Ursulin019bf272016-07-13 16:03:41 +0100429/**
430 * intel_engines_setup_common - setup engine state not requiring hw access
431 * @engine: Engine to setup.
432 *
433 * Initializes @engine@ structure members shared between legacy and execlists
434 * submission modes which do not require hardware access.
435 *
436 * Typically done early in the submission mode specific engine setup stage.
437 */
438void intel_engine_setup_common(struct intel_engine_cs *engine)
439{
Mika Kuoppala19df9a52017-09-22 15:43:04 +0300440 intel_engine_init_execlist(engine);
Tvrtko Ursulin019bf272016-07-13 16:03:41 +0100441
Chris Wilson73cb9702016-10-28 13:58:46 +0100442 intel_engine_init_timeline(engine);
Tvrtko Ursulin019bf272016-07-13 16:03:41 +0100443 intel_engine_init_hangcheck(engine);
Chris Wilson115003e92016-08-04 16:32:19 +0100444 i915_gem_batch_pool_init(engine, &engine->batch_pool);
Chris Wilson7756e452016-08-18 17:17:10 +0100445
446 intel_engine_init_cmd_parser(engine);
Tvrtko Ursulin019bf272016-07-13 16:03:41 +0100447}
448
Chris Wilsonadc320c2016-08-15 10:48:59 +0100449int intel_engine_create_scratch(struct intel_engine_cs *engine, int size)
450{
451 struct drm_i915_gem_object *obj;
452 struct i915_vma *vma;
453 int ret;
454
455 WARN_ON(engine->scratch);
456
Tvrtko Ursulin187685c2016-12-01 14:16:36 +0000457 obj = i915_gem_object_create_stolen(engine->i915, size);
Chris Wilsonadc320c2016-08-15 10:48:59 +0100458 if (!obj)
Chris Wilson920cf412016-10-28 13:58:30 +0100459 obj = i915_gem_object_create_internal(engine->i915, size);
Chris Wilsonadc320c2016-08-15 10:48:59 +0100460 if (IS_ERR(obj)) {
461 DRM_ERROR("Failed to allocate scratch page\n");
462 return PTR_ERR(obj);
463 }
464
Chris Wilsona01cb372017-01-16 15:21:30 +0000465 vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
Chris Wilsonadc320c2016-08-15 10:48:59 +0100466 if (IS_ERR(vma)) {
467 ret = PTR_ERR(vma);
468 goto err_unref;
469 }
470
471 ret = i915_vma_pin(vma, 0, 4096, PIN_GLOBAL | PIN_HIGH);
472 if (ret)
473 goto err_unref;
474
475 engine->scratch = vma;
Chris Wilsonbde13eb2016-08-15 10:49:07 +0100476 DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
477 engine->name, i915_ggtt_offset(vma));
Chris Wilsonadc320c2016-08-15 10:48:59 +0100478 return 0;
479
480err_unref:
481 i915_gem_object_put(obj);
482 return ret;
483}
484
485static void intel_engine_cleanup_scratch(struct intel_engine_cs *engine)
486{
Chris Wilson19880c42016-08-15 10:49:05 +0100487 i915_vma_unpin_and_release(&engine->scratch);
Chris Wilsonadc320c2016-08-15 10:48:59 +0100488}
489
Daniele Ceraolo Spurio486e93f2017-09-13 09:56:02 +0100490static void cleanup_phys_status_page(struct intel_engine_cs *engine)
491{
492 struct drm_i915_private *dev_priv = engine->i915;
493
494 if (!dev_priv->status_page_dmah)
495 return;
496
497 drm_pci_free(&dev_priv->drm, dev_priv->status_page_dmah);
498 engine->status_page.page_addr = NULL;
499}
500
501static void cleanup_status_page(struct intel_engine_cs *engine)
502{
503 struct i915_vma *vma;
504 struct drm_i915_gem_object *obj;
505
506 vma = fetch_and_zero(&engine->status_page.vma);
507 if (!vma)
508 return;
509
510 obj = vma->obj;
511
512 i915_vma_unpin(vma);
513 i915_vma_close(vma);
514
515 i915_gem_object_unpin_map(obj);
516 __i915_gem_object_release_unless_active(obj);
517}
518
519static int init_status_page(struct intel_engine_cs *engine)
520{
521 struct drm_i915_gem_object *obj;
522 struct i915_vma *vma;
523 unsigned int flags;
524 void *vaddr;
525 int ret;
526
527 obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
528 if (IS_ERR(obj)) {
529 DRM_ERROR("Failed to allocate status page\n");
530 return PTR_ERR(obj);
531 }
532
533 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
534 if (ret)
535 goto err;
536
537 vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
538 if (IS_ERR(vma)) {
539 ret = PTR_ERR(vma);
540 goto err;
541 }
542
543 flags = PIN_GLOBAL;
544 if (!HAS_LLC(engine->i915))
545 /* On g33, we cannot place HWS above 256MiB, so
546 * restrict its pinning to the low mappable arena.
547 * Though this restriction is not documented for
548 * gen4, gen5, or byt, they also behave similarly
549 * and hang if the HWS is placed at the top of the
550 * GTT. To generalise, it appears that all !llc
551 * platforms have issues with us placing the HWS
552 * above the mappable region (even though we never
553 * actually map it).
554 */
555 flags |= PIN_MAPPABLE;
Chris Wilson34a04e52017-09-13 09:56:03 +0100556 else
557 flags |= PIN_HIGH;
Daniele Ceraolo Spurio486e93f2017-09-13 09:56:02 +0100558 ret = i915_vma_pin(vma, 0, 4096, flags);
559 if (ret)
560 goto err;
561
562 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
563 if (IS_ERR(vaddr)) {
564 ret = PTR_ERR(vaddr);
565 goto err_unpin;
566 }
567
568 engine->status_page.vma = vma;
569 engine->status_page.ggtt_offset = i915_ggtt_offset(vma);
570 engine->status_page.page_addr = memset(vaddr, 0, PAGE_SIZE);
571
572 DRM_DEBUG_DRIVER("%s hws offset: 0x%08x\n",
573 engine->name, i915_ggtt_offset(vma));
574 return 0;
575
576err_unpin:
577 i915_vma_unpin(vma);
578err:
579 i915_gem_object_put(obj);
580 return ret;
581}
582
583static int init_phys_status_page(struct intel_engine_cs *engine)
584{
585 struct drm_i915_private *dev_priv = engine->i915;
586
587 GEM_BUG_ON(engine->id != RCS);
588
589 dev_priv->status_page_dmah =
590 drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
591 if (!dev_priv->status_page_dmah)
592 return -ENOMEM;
593
594 engine->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
595 memset(engine->status_page.page_addr, 0, PAGE_SIZE);
596
597 return 0;
598}
599
Tvrtko Ursulin019bf272016-07-13 16:03:41 +0100600/**
601 * intel_engines_init_common - initialize cengine state which might require hw access
602 * @engine: Engine to initialize.
603 *
604 * Initializes @engine@ structure members shared between legacy and execlists
605 * submission modes which do require hardware access.
606 *
607 * Typcally done at later stages of submission mode specific engine setup.
608 *
609 * Returns zero on success or an error code on failure.
610 */
611int intel_engine_init_common(struct intel_engine_cs *engine)
612{
Chris Wilson266a2402017-05-04 10:33:08 +0100613 struct intel_ring *ring;
Tvrtko Ursulin019bf272016-07-13 16:03:41 +0100614 int ret;
615
Chris Wilsonff44ad52017-03-16 17:13:03 +0000616 engine->set_default_submission(engine);
617
Chris Wilsone8a9c582016-12-18 15:37:20 +0000618 /* We may need to do things with the shrinker which
619 * require us to immediately switch back to the default
620 * context. This can cause a problem as pinning the
621 * default context also requires GTT space which may not
622 * be available. To avoid this we always pin the default
623 * context.
624 */
Chris Wilson266a2402017-05-04 10:33:08 +0100625 ring = engine->context_pin(engine, engine->i915->kernel_context);
626 if (IS_ERR(ring))
627 return PTR_ERR(ring);
Tvrtko Ursulin019bf272016-07-13 16:03:41 +0100628
Chris Wilsone7af3112017-10-03 21:34:48 +0100629 /*
630 * Similarly the preempt context must always be available so that
631 * we can interrupt the engine at any time.
632 */
Michał Winiarskia4598d12017-10-25 22:00:18 +0200633 if (HAS_LOGICAL_RING_PREEMPTION(engine->i915)) {
Chris Wilsone7af3112017-10-03 21:34:48 +0100634 ring = engine->context_pin(engine,
635 engine->i915->preempt_context);
636 if (IS_ERR(ring)) {
637 ret = PTR_ERR(ring);
638 goto err_unpin_kernel;
639 }
640 }
641
Chris Wilsone8a9c582016-12-18 15:37:20 +0000642 ret = intel_engine_init_breadcrumbs(engine);
643 if (ret)
Chris Wilsone7af3112017-10-03 21:34:48 +0100644 goto err_unpin_preempt;
Chris Wilsone8a9c582016-12-18 15:37:20 +0000645
Daniele Ceraolo Spurio486e93f2017-09-13 09:56:02 +0100646 if (HWS_NEEDS_PHYSICAL(engine->i915))
647 ret = init_phys_status_page(engine);
648 else
649 ret = init_status_page(engine);
650 if (ret)
Chris Wilson7c2fa7f2017-11-10 14:26:34 +0000651 goto err_breadcrumbs;
Chris Wilson4e50f082016-10-28 13:58:31 +0100652
Chris Wilson7756e452016-08-18 17:17:10 +0100653 return 0;
Chris Wilsone8a9c582016-12-18 15:37:20 +0000654
Daniele Ceraolo Spurio486e93f2017-09-13 09:56:02 +0100655err_breadcrumbs:
656 intel_engine_fini_breadcrumbs(engine);
Chris Wilsone7af3112017-10-03 21:34:48 +0100657err_unpin_preempt:
Michał Winiarskia4598d12017-10-25 22:00:18 +0200658 if (HAS_LOGICAL_RING_PREEMPTION(engine->i915))
Chris Wilsone7af3112017-10-03 21:34:48 +0100659 engine->context_unpin(engine, engine->i915->preempt_context);
660err_unpin_kernel:
Chris Wilsone8a9c582016-12-18 15:37:20 +0000661 engine->context_unpin(engine, engine->i915->kernel_context);
662 return ret;
Tvrtko Ursulin019bf272016-07-13 16:03:41 +0100663}
Chris Wilson96a945a2016-08-03 13:19:16 +0100664
665/**
666 * intel_engines_cleanup_common - cleans up the engine state created by
667 * the common initiailizers.
668 * @engine: Engine to cleanup.
669 *
670 * This cleans up everything created by the common helpers.
671 */
672void intel_engine_cleanup_common(struct intel_engine_cs *engine)
673{
Chris Wilsonadc320c2016-08-15 10:48:59 +0100674 intel_engine_cleanup_scratch(engine);
675
Daniele Ceraolo Spurio486e93f2017-09-13 09:56:02 +0100676 if (HWS_NEEDS_PHYSICAL(engine->i915))
677 cleanup_phys_status_page(engine);
678 else
679 cleanup_status_page(engine);
680
Chris Wilson96a945a2016-08-03 13:19:16 +0100681 intel_engine_fini_breadcrumbs(engine);
Chris Wilson7756e452016-08-18 17:17:10 +0100682 intel_engine_cleanup_cmd_parser(engine);
Chris Wilson96a945a2016-08-03 13:19:16 +0100683 i915_gem_batch_pool_fini(&engine->batch_pool);
Chris Wilsone8a9c582016-12-18 15:37:20 +0000684
Chris Wilsond2b4b972017-11-10 14:26:33 +0000685 if (engine->default_state)
686 i915_gem_object_put(engine->default_state);
687
Michał Winiarskia4598d12017-10-25 22:00:18 +0200688 if (HAS_LOGICAL_RING_PREEMPTION(engine->i915))
Chris Wilsone7af3112017-10-03 21:34:48 +0100689 engine->context_unpin(engine, engine->i915->preempt_context);
Chris Wilsone8a9c582016-12-18 15:37:20 +0000690 engine->context_unpin(engine, engine->i915->kernel_context);
Chris Wilson96a945a2016-08-03 13:19:16 +0100691}
Chris Wilson1b365952016-10-04 21:11:31 +0100692
693u64 intel_engine_get_active_head(struct intel_engine_cs *engine)
694{
695 struct drm_i915_private *dev_priv = engine->i915;
696 u64 acthd;
697
698 if (INTEL_GEN(dev_priv) >= 8)
699 acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
700 RING_ACTHD_UDW(engine->mmio_base));
701 else if (INTEL_GEN(dev_priv) >= 4)
702 acthd = I915_READ(RING_ACTHD(engine->mmio_base));
703 else
704 acthd = I915_READ(ACTHD);
705
706 return acthd;
707}
708
709u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine)
710{
711 struct drm_i915_private *dev_priv = engine->i915;
712 u64 bbaddr;
713
714 if (INTEL_GEN(dev_priv) >= 8)
715 bbaddr = I915_READ64_2x32(RING_BBADDR(engine->mmio_base),
716 RING_BBADDR_UDW(engine->mmio_base));
717 else
718 bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
719
720 return bbaddr;
721}
Chris Wilson0e704472016-10-12 10:05:17 +0100722
723const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
724{
725 switch (type) {
726 case I915_CACHE_NONE: return " uncached";
727 case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
728 case I915_CACHE_L3_LLC: return " L3+LLC";
729 case I915_CACHE_WT: return " WT";
730 default: return "";
731 }
732}
733
734static inline uint32_t
735read_subslice_reg(struct drm_i915_private *dev_priv, int slice,
736 int subslice, i915_reg_t reg)
737{
738 uint32_t mcr;
739 uint32_t ret;
740 enum forcewake_domains fw_domains;
741
742 fw_domains = intel_uncore_forcewake_for_reg(dev_priv, reg,
743 FW_REG_READ);
744 fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
745 GEN8_MCR_SELECTOR,
746 FW_REG_READ | FW_REG_WRITE);
747
748 spin_lock_irq(&dev_priv->uncore.lock);
749 intel_uncore_forcewake_get__locked(dev_priv, fw_domains);
750
751 mcr = I915_READ_FW(GEN8_MCR_SELECTOR);
752 /*
753 * The HW expects the slice and sublice selectors to be reset to 0
754 * after reading out the registers.
755 */
756 WARN_ON_ONCE(mcr & (GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK));
757 mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
758 mcr |= GEN8_MCR_SLICE(slice) | GEN8_MCR_SUBSLICE(subslice);
759 I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
760
761 ret = I915_READ_FW(reg);
762
763 mcr &= ~(GEN8_MCR_SLICE_MASK | GEN8_MCR_SUBSLICE_MASK);
764 I915_WRITE_FW(GEN8_MCR_SELECTOR, mcr);
765
766 intel_uncore_forcewake_put__locked(dev_priv, fw_domains);
767 spin_unlock_irq(&dev_priv->uncore.lock);
768
769 return ret;
770}
771
772/* NB: please notice the memset */
773void intel_engine_get_instdone(struct intel_engine_cs *engine,
774 struct intel_instdone *instdone)
775{
776 struct drm_i915_private *dev_priv = engine->i915;
777 u32 mmio_base = engine->mmio_base;
778 int slice;
779 int subslice;
780
781 memset(instdone, 0, sizeof(*instdone));
782
783 switch (INTEL_GEN(dev_priv)) {
784 default:
785 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
786
787 if (engine->id != RCS)
788 break;
789
790 instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
791 for_each_instdone_slice_subslice(dev_priv, slice, subslice) {
792 instdone->sampler[slice][subslice] =
793 read_subslice_reg(dev_priv, slice, subslice,
794 GEN7_SAMPLER_INSTDONE);
795 instdone->row[slice][subslice] =
796 read_subslice_reg(dev_priv, slice, subslice,
797 GEN7_ROW_INSTDONE);
798 }
799 break;
800 case 7:
801 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
802
803 if (engine->id != RCS)
804 break;
805
806 instdone->slice_common = I915_READ(GEN7_SC_INSTDONE);
807 instdone->sampler[0][0] = I915_READ(GEN7_SAMPLER_INSTDONE);
808 instdone->row[0][0] = I915_READ(GEN7_ROW_INSTDONE);
809
810 break;
811 case 6:
812 case 5:
813 case 4:
814 instdone->instdone = I915_READ(RING_INSTDONE(mmio_base));
815
816 if (engine->id == RCS)
817 /* HACK: Using the wrong struct member */
818 instdone->slice_common = I915_READ(GEN4_INSTDONE1);
819 break;
820 case 3:
821 case 2:
822 instdone->instdone = I915_READ(GEN2_INSTDONE);
823 break;
824 }
825}
Chris Wilsonf97fbf92017-02-13 17:15:14 +0000826
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +0000827static int wa_add(struct drm_i915_private *dev_priv,
828 i915_reg_t addr,
829 const u32 mask, const u32 val)
830{
831 const u32 idx = dev_priv->workarounds.count;
832
833 if (WARN_ON(idx >= I915_MAX_WA_REGS))
834 return -ENOSPC;
835
836 dev_priv->workarounds.reg[idx].addr = addr;
837 dev_priv->workarounds.reg[idx].value = val;
838 dev_priv->workarounds.reg[idx].mask = mask;
839
840 dev_priv->workarounds.count++;
841
842 return 0;
843}
844
845#define WA_REG(addr, mask, val) do { \
846 const int r = wa_add(dev_priv, (addr), (mask), (val)); \
847 if (r) \
848 return r; \
849 } while (0)
850
851#define WA_SET_BIT_MASKED(addr, mask) \
852 WA_REG(addr, (mask), _MASKED_BIT_ENABLE(mask))
853
854#define WA_CLR_BIT_MASKED(addr, mask) \
855 WA_REG(addr, (mask), _MASKED_BIT_DISABLE(mask))
856
857#define WA_SET_FIELD_MASKED(addr, mask, value) \
858 WA_REG(addr, mask, _MASKED_FIELD(mask, value))
859
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +0000860static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
861 i915_reg_t reg)
862{
863 struct drm_i915_private *dev_priv = engine->i915;
864 struct i915_workarounds *wa = &dev_priv->workarounds;
865 const uint32_t index = wa->hw_whitelist_count[engine->id];
866
867 if (WARN_ON(index >= RING_MAX_NONPRIV_SLOTS))
868 return -EINVAL;
869
Oscar Mateo32ced392017-09-28 15:40:39 -0700870 I915_WRITE(RING_FORCE_TO_NONPRIV(engine->mmio_base, index),
871 i915_mmio_reg_offset(reg));
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +0000872 wa->hw_whitelist_count[engine->id]++;
873
874 return 0;
875}
876
877static int gen8_init_workarounds(struct intel_engine_cs *engine)
878{
879 struct drm_i915_private *dev_priv = engine->i915;
880
881 WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
882
883 /* WaDisableAsyncFlipPerfMode:bdw,chv */
884 WA_SET_BIT_MASKED(MI_MODE, ASYNC_FLIP_PERF_DISABLE);
885
886 /* WaDisablePartialInstShootdown:bdw,chv */
887 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
888 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
889
890 /* Use Force Non-Coherent whenever executing a 3D context. This is a
891 * workaround for for a possible hang in the unlikely event a TLB
892 * invalidation occurs during a PSD flush.
893 */
894 /* WaForceEnableNonCoherent:bdw,chv */
895 /* WaHdcDisableFetchWhenMasked:bdw,chv */
896 WA_SET_BIT_MASKED(HDC_CHICKEN0,
897 HDC_DONOT_FETCH_MEM_WHEN_MASKED |
898 HDC_FORCE_NON_COHERENT);
899
900 /* From the Haswell PRM, Command Reference: Registers, CACHE_MODE_0:
901 * "The Hierarchical Z RAW Stall Optimization allows non-overlapping
902 * polygons in the same 8x4 pixel/sample area to be processed without
903 * stalling waiting for the earlier ones to write to Hierarchical Z
904 * buffer."
905 *
906 * This optimization is off by default for BDW and CHV; turn it on.
907 */
908 WA_CLR_BIT_MASKED(CACHE_MODE_0_GEN7, HIZ_RAW_STALL_OPT_DISABLE);
909
910 /* Wa4x4STCOptimizationDisable:bdw,chv */
911 WA_SET_BIT_MASKED(CACHE_MODE_1, GEN8_4x4_STC_OPTIMIZATION_DISABLE);
912
913 /*
914 * BSpec recommends 8x4 when MSAA is used,
915 * however in practice 16x4 seems fastest.
916 *
917 * Note that PS/WM thread counts depend on the WIZ hashing
918 * disable bit, which we don't touch here, but it's good
919 * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
920 */
921 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
922 GEN6_WIZ_HASHING_MASK,
923 GEN6_WIZ_HASHING_16x4);
924
925 return 0;
926}
927
928static int bdw_init_workarounds(struct intel_engine_cs *engine)
929{
930 struct drm_i915_private *dev_priv = engine->i915;
931 int ret;
932
933 ret = gen8_init_workarounds(engine);
934 if (ret)
935 return ret;
936
937 /* WaDisableThreadStallDopClockGating:bdw (pre-production) */
938 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
939
940 /* WaDisableDopClockGating:bdw
941 *
942 * Also see the related UCGTCL1 write in broadwell_init_clock_gating()
943 * to disable EUTC clock gating.
944 */
945 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2,
946 DOP_CLOCK_GATING_DISABLE);
947
948 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
949 GEN8_SAMPLER_POWER_BYPASS_DIS);
950
951 WA_SET_BIT_MASKED(HDC_CHICKEN0,
952 /* WaForceContextSaveRestoreNonCoherent:bdw */
953 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
954 /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
955 (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
956
957 return 0;
958}
959
960static int chv_init_workarounds(struct intel_engine_cs *engine)
961{
962 struct drm_i915_private *dev_priv = engine->i915;
963 int ret;
964
965 ret = gen8_init_workarounds(engine);
966 if (ret)
967 return ret;
968
969 /* WaDisableThreadStallDopClockGating:chv */
970 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, STALL_DOP_GATING_DISABLE);
971
972 /* Improve HiZ throughput on CHV. */
973 WA_SET_BIT_MASKED(HIZ_CHICKEN, CHV_HZ_8X8_MODE_IN_1X);
974
975 return 0;
976}
977
978static int gen9_init_workarounds(struct intel_engine_cs *engine)
979{
980 struct drm_i915_private *dev_priv = engine->i915;
981 int ret;
982
Rodrigo Vivi46c26662017-06-16 15:49:58 -0700983 /* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl,glk,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +0000984 I915_WRITE(GEN9_CSFE_CHICKEN1_RCS, _MASKED_BIT_ENABLE(GEN9_PREEMPT_GPGPU_SYNC_SWITCH_DISABLE));
985
Rodrigo Vivi46c26662017-06-16 15:49:58 -0700986 /* WaEnableLbsSlaRetryTimerDecrement:skl,bxt,kbl,glk,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +0000987 I915_WRITE(BDW_SCRATCH1, I915_READ(BDW_SCRATCH1) |
988 GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE);
989
Rodrigo Vivi98eed3d2017-06-19 14:21:47 -0700990 /* WaDisableKillLogic:bxt,skl,kbl */
991 if (!IS_COFFEELAKE(dev_priv))
992 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
993 ECOCHK_DIS_TLB);
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +0000994
Ville Syrjälä93564042017-08-24 22:10:51 +0300995 if (HAS_LLC(dev_priv)) {
996 /* WaCompressedResourceSamplerPbeMediaNewHashMode:skl,kbl
997 *
998 * Must match Display Engine. See
999 * WaCompressedResourceDisplayNewHashMode.
1000 */
1001 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1002 GEN9_PBE_COMPRESSED_HASH_SELECTION);
1003 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
1004 GEN9_SAMPLER_HASH_COMPRESSED_READ_ADDR);
Chris Wilson53221e12017-10-04 13:41:52 +01001005
1006 I915_WRITE(MMCD_MISC_CTRL,
1007 I915_READ(MMCD_MISC_CTRL) |
1008 MMCD_PCLA |
1009 MMCD_HOTSPOT_EN);
Ville Syrjälä93564042017-08-24 22:10:51 +03001010 }
1011
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001012 /* WaClearFlowControlGpgpuContextSave:skl,bxt,kbl,glk,cfl */
1013 /* WaDisablePartialInstShootdown:skl,bxt,kbl,glk,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001014 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1015 FLOW_CONTROL_ENABLE |
1016 PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE);
1017
1018 /* Syncing dependencies between camera and graphics:skl,bxt,kbl */
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001019 if (!IS_COFFEELAKE(dev_priv))
1020 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
1021 GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001022
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001023 /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
1024 /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001025 WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
Arkadiusz Hiler0b71cea2017-05-12 13:20:15 +02001026 GEN9_ENABLE_YV12_BUGFIX |
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001027 GEN9_ENABLE_GPGPU_PREEMPTION);
1028
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001029 /* Wa4x4STCOptimizationDisable:skl,bxt,kbl,glk,cfl */
1030 /* WaDisablePartialResolveInVc:skl,bxt,kbl,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001031 WA_SET_BIT_MASKED(CACHE_MODE_1, (GEN8_4x4_STC_OPTIMIZATION_DISABLE |
1032 GEN9_PARTIAL_RESOLVE_IN_VC_DISABLE));
1033
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001034 /* WaCcsTlbPrefetchDisable:skl,bxt,kbl,glk,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001035 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
1036 GEN9_CCS_TLB_PREFETCH_ENABLE);
1037
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001038 /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001039 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1040 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
1041 HDC_FORCE_CSR_NON_COHERENT_OVR_DISABLE);
1042
1043 /* WaForceEnableNonCoherent and WaDisableHDCInvalidation are
1044 * both tied to WaForceContextSaveRestoreNonCoherent
1045 * in some hsds for skl. We keep the tie for all gen9. The
1046 * documentation is a bit hazy and so we want to get common behaviour,
1047 * even though there is no clear evidence we would need both on kbl/bxt.
1048 * This area has been source of system hangs so we play it safe
1049 * and mimic the skl regardless of what bspec says.
1050 *
1051 * Use Force Non-Coherent whenever executing a 3D context. This
1052 * is a workaround for a possible hang in the unlikely event
1053 * a TLB invalidation occurs during a PSD flush.
1054 */
1055
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001056 /* WaForceEnableNonCoherent:skl,bxt,kbl,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001057 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1058 HDC_FORCE_NON_COHERENT);
1059
Rodrigo Vivi98eed3d2017-06-19 14:21:47 -07001060 /* WaDisableHDCInvalidation:skl,bxt,kbl,cfl */
1061 I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) |
1062 BDW_DISABLE_HDC_INVALIDATION);
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001063
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001064 /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001065 if (IS_SKYLAKE(dev_priv) ||
1066 IS_KABYLAKE(dev_priv) ||
Chris Wilsonf3e2b2c2017-11-14 13:43:39 +00001067 IS_COFFEELAKE(dev_priv))
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001068 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
1069 GEN8_SAMPLER_POWER_BYPASS_DIS);
1070
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001071 /* WaDisableSTUnitPowerOptimization:skl,bxt,kbl,glk,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001072 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN2, GEN8_ST_PO_DISABLE);
1073
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001074 /* WaOCLCoherentLineFlush:skl,bxt,kbl,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001075 I915_WRITE(GEN8_L3SQCREG4, (I915_READ(GEN8_L3SQCREG4) |
1076 GEN8_LQSC_FLUSH_COHERENT_LINES));
1077
Michał Winiarski5152def2017-10-03 21:34:46 +01001078 /*
1079 * Supporting preemption with fine-granularity requires changes in the
1080 * batch buffer programming. Since we can't break old userspace, we
1081 * need to set our default preemption level to safe value. Userspace is
1082 * still able to use more fine-grained preemption levels, since in
1083 * WaEnablePreemptionGranularityControlByUMD we're whitelisting the
1084 * per-ctx register. As such, WaDisable{3D,GPGPU}MidCmdPreemption are
1085 * not real HW workarounds, but merely a way to start using preemption
1086 * while maintaining old contract with userspace.
1087 */
1088
1089 /* WaDisable3DMidCmdPreemption:skl,bxt,glk,cfl,[cnl] */
1090 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
1091
1092 /* WaDisableGPGPUMidCmdPreemption:skl,bxt,blk,cfl,[cnl] */
1093 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK,
1094 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
1095
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001096 /* WaVFEStateAfterPipeControlwithMediaStateClear:skl,bxt,glk,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001097 ret = wa_ring_whitelist_reg(engine, GEN9_CTX_PREEMPT_REG);
1098 if (ret)
1099 return ret;
1100
Jeff McGee1e998342017-10-03 21:34:45 +01001101 /* WaEnablePreemptionGranularityControlByUMD:skl,bxt,kbl,cfl,[cnl] */
1102 I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
1103 _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
1104 ret = wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001105 if (ret)
1106 return ret;
1107
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001108 /* WaAllowUMDToModifyHDCChicken1:skl,bxt,kbl,glk,cfl */
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001109 ret = wa_ring_whitelist_reg(engine, GEN8_HDC_CHICKEN1);
1110 if (ret)
1111 return ret;
1112
1113 return 0;
1114}
1115
1116static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
1117{
1118 struct drm_i915_private *dev_priv = engine->i915;
1119 u8 vals[3] = { 0, 0, 0 };
1120 unsigned int i;
1121
1122 for (i = 0; i < 3; i++) {
1123 u8 ss;
1124
1125 /*
1126 * Only consider slices where one, and only one, subslice has 7
1127 * EUs
1128 */
1129 if (!is_power_of_2(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]))
1130 continue;
1131
1132 /*
1133 * subslice_7eu[i] != 0 (because of the check above) and
1134 * ss_max == 4 (maximum number of subslices possible per slice)
1135 *
1136 * -> 0 <= ss <= 3;
1137 */
1138 ss = ffs(INTEL_INFO(dev_priv)->sseu.subslice_7eu[i]) - 1;
1139 vals[i] = 3 - ss;
1140 }
1141
1142 if (vals[0] == 0 && vals[1] == 0 && vals[2] == 0)
1143 return 0;
1144
1145 /* Tune IZ hashing. See intel_device_info_runtime_init() */
1146 WA_SET_FIELD_MASKED(GEN7_GT_MODE,
1147 GEN9_IZ_HASHING_MASK(2) |
1148 GEN9_IZ_HASHING_MASK(1) |
1149 GEN9_IZ_HASHING_MASK(0),
1150 GEN9_IZ_HASHING(2, vals[2]) |
1151 GEN9_IZ_HASHING(1, vals[1]) |
1152 GEN9_IZ_HASHING(0, vals[0]));
1153
1154 return 0;
1155}
1156
1157static int skl_init_workarounds(struct intel_engine_cs *engine)
1158{
1159 struct drm_i915_private *dev_priv = engine->i915;
1160 int ret;
1161
1162 ret = gen9_init_workarounds(engine);
1163 if (ret)
1164 return ret;
1165
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001166 /* WaEnableGapsTsvCreditFix:skl */
1167 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1168 GEN9_GAPS_TSV_CREDIT_DISABLE));
1169
1170 /* WaDisableGafsUnitClkGating:skl */
Oscar Mateo4827c542017-09-07 08:40:07 -07001171 I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
1172 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001173
1174 /* WaInPlaceDecompressionHang:skl */
1175 if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
Oscar Mateoefc886c2017-09-07 08:40:04 -07001176 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1177 (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1178 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001179
1180 /* WaDisableLSQCROPERFforOCL:skl */
1181 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1182 if (ret)
1183 return ret;
1184
1185 return skl_tune_iz_hashing(engine);
1186}
1187
1188static int bxt_init_workarounds(struct intel_engine_cs *engine)
1189{
1190 struct drm_i915_private *dev_priv = engine->i915;
Chris Wilson70a84f32017-11-14 13:43:40 +00001191 u32 val;
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001192 int ret;
1193
1194 ret = gen9_init_workarounds(engine);
1195 if (ret)
1196 return ret;
1197
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001198 /* WaDisableThreadStallDopClockGating:bxt */
1199 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
1200 STALL_DOP_GATING_DISABLE);
1201
1202 /* WaDisablePooledEuLoadBalancingFix:bxt */
Chris Wilson70a84f32017-11-14 13:43:40 +00001203 I915_WRITE(FF_SLICE_CS_CHICKEN2,
1204 _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001205
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001206 /* WaProgramL3SqcReg1DefaultForPerf:bxt */
Chris Wilson70a84f32017-11-14 13:43:40 +00001207 val = I915_READ(GEN8_L3SQCREG1);
1208 val &= ~L3_PRIO_CREDITS_MASK;
1209 val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2);
1210 I915_WRITE(GEN8_L3SQCREG1, val);
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001211
1212 /* WaToEnableHwFixForPushConstHWBug:bxt */
Chris Wilson70a84f32017-11-14 13:43:40 +00001213 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1214 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001215
1216 /* WaInPlaceDecompressionHang:bxt */
Chris Wilson70a84f32017-11-14 13:43:40 +00001217 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1218 (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1219 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001220
1221 return 0;
1222}
1223
Rodrigo Vivi90007bc2017-08-15 16:16:48 -07001224static int cnl_init_workarounds(struct intel_engine_cs *engine)
1225{
1226 struct drm_i915_private *dev_priv = engine->i915;
1227 int ret;
1228
Oscar Mateo6cf20a02017-09-07 08:40:05 -07001229 /* WaDisableI2mCycleOnWRPort:cnl (pre-prod) */
Rodrigo Vivi86ebb012017-08-29 16:07:51 -07001230 if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
Oscar Mateo6cf20a02017-09-07 08:40:05 -07001231 I915_WRITE(GAMT_CHKN_BIT_REG,
1232 (I915_READ(GAMT_CHKN_BIT_REG) |
1233 GAMT_CHKN_DISABLE_I2M_CYCLE_ON_WR_PORT));
Rodrigo Vivi86ebb012017-08-29 16:07:51 -07001234
Rodrigo Viviacfb5552017-08-23 13:35:04 -07001235 /* WaForceContextSaveRestoreNonCoherent:cnl */
1236 WA_SET_BIT_MASKED(CNL_HDC_CHICKEN0,
1237 HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT);
1238
Rodrigo Viviaa9f4c42017-09-06 15:03:25 -07001239 /* WaThrottleEUPerfToAvoidTDBackPressure:cnl(pre-prod) */
1240 if (IS_CNL_REVID(dev_priv, CNL_REVID_B0, CNL_REVID_B0))
1241 WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN, THROTTLE_12_5);
1242
Rodrigo Vivie6d1a4f2017-08-15 16:16:49 -07001243 /* WaDisableReplayBufferBankArbitrationOptimization:cnl */
1244 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1245 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1246
Rodrigo Vivid1d24752017-08-15 16:16:50 -07001247 /* WaDisableEnhancedSBEVertexCaching:cnl (pre-prod) */
1248 if (IS_CNL_REVID(dev_priv, 0, CNL_REVID_B0))
1249 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1250 GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE);
1251
Rodrigo Vivi90007bc2017-08-15 16:16:48 -07001252 /* WaInPlaceDecompressionHang:cnl */
Oscar Mateoefc886c2017-09-07 08:40:04 -07001253 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1254 (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1255 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
Rodrigo Vivi90007bc2017-08-15 16:16:48 -07001256
Oscar Mateo2cbecff2017-08-23 12:56:31 -07001257 /* WaPushConstantDereferenceHoldDisable:cnl */
Oscar Mateob27f5902017-09-07 08:40:06 -07001258 WA_SET_BIT_MASKED(GEN7_ROW_CHICKEN2, PUSH_CONSTANT_DEREF_DISABLE);
Oscar Mateo2cbecff2017-08-23 12:56:31 -07001259
Rodrigo Vivi392572f2017-08-29 16:07:23 -07001260 /* FtrEnableFastAnisoL1BankingFix: cnl */
1261 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3, CNL_FAST_ANISO_L1_BANKING_FIX);
1262
Michał Winiarski5152def2017-10-03 21:34:46 +01001263 /* WaDisable3DMidCmdPreemption:cnl */
1264 WA_CLR_BIT_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_3D_OBJECT_LEVEL);
1265
1266 /* WaDisableGPGPUMidCmdPreemption:cnl */
1267 WA_SET_FIELD_MASKED(GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK,
1268 GEN9_PREEMPT_GPGPU_COMMAND_LEVEL);
1269
Rodrigo Vivi90007bc2017-08-15 16:16:48 -07001270 /* WaEnablePreemptionGranularityControlByUMD:cnl */
Jeff McGee1e998342017-10-03 21:34:45 +01001271 I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
1272 _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
Rodrigo Vivi90007bc2017-08-15 16:16:48 -07001273 ret= wa_ring_whitelist_reg(engine, GEN8_CS_CHICKEN1);
1274 if (ret)
1275 return ret;
1276
1277 return 0;
1278}
1279
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001280static int kbl_init_workarounds(struct intel_engine_cs *engine)
1281{
1282 struct drm_i915_private *dev_priv = engine->i915;
1283 int ret;
1284
1285 ret = gen9_init_workarounds(engine);
1286 if (ret)
1287 return ret;
1288
1289 /* WaEnableGapsTsvCreditFix:kbl */
1290 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1291 GEN9_GAPS_TSV_CREDIT_DISABLE));
1292
1293 /* WaDisableDynamicCreditSharing:kbl */
1294 if (IS_KBL_REVID(dev_priv, 0, KBL_REVID_B0))
Oscar Mateoc6ea497c2017-09-07 08:40:08 -07001295 I915_WRITE(GAMT_CHKN_BIT_REG,
1296 (I915_READ(GAMT_CHKN_BIT_REG) |
1297 GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING));
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001298
1299 /* WaDisableFenceDestinationToSLM:kbl (pre-prod) */
1300 if (IS_KBL_REVID(dev_priv, KBL_REVID_A0, KBL_REVID_A0))
1301 WA_SET_BIT_MASKED(HDC_CHICKEN0,
1302 HDC_FENCE_DEST_SLM_DISABLE);
1303
1304 /* WaToEnableHwFixForPushConstHWBug:kbl */
1305 if (IS_KBL_REVID(dev_priv, KBL_REVID_C0, REVID_FOREVER))
1306 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1307 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1308
1309 /* WaDisableGafsUnitClkGating:kbl */
Oscar Mateo4827c542017-09-07 08:40:07 -07001310 I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
1311 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001312
1313 /* WaDisableSbeCacheDispatchPortSharing:kbl */
1314 WA_SET_BIT_MASKED(
1315 GEN7_HALF_SLICE_CHICKEN1,
1316 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1317
1318 /* WaInPlaceDecompressionHang:kbl */
Oscar Mateoefc886c2017-09-07 08:40:04 -07001319 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1320 (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1321 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001322
1323 /* WaDisableLSQCROPERFforOCL:kbl */
1324 ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
1325 if (ret)
1326 return ret;
1327
1328 return 0;
1329}
1330
1331static int glk_init_workarounds(struct intel_engine_cs *engine)
1332{
1333 struct drm_i915_private *dev_priv = engine->i915;
1334 int ret;
1335
1336 ret = gen9_init_workarounds(engine);
1337 if (ret)
1338 return ret;
1339
1340 /* WaToEnableHwFixForPushConstHWBug:glk */
1341 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1342 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1343
1344 return 0;
1345}
1346
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001347static int cfl_init_workarounds(struct intel_engine_cs *engine)
1348{
1349 struct drm_i915_private *dev_priv = engine->i915;
1350 int ret;
1351
1352 ret = gen9_init_workarounds(engine);
1353 if (ret)
1354 return ret;
1355
1356 /* WaEnableGapsTsvCreditFix:cfl */
1357 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
1358 GEN9_GAPS_TSV_CREDIT_DISABLE));
1359
1360 /* WaToEnableHwFixForPushConstHWBug:cfl */
1361 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
1362 GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
1363
1364 /* WaDisableGafsUnitClkGating:cfl */
Oscar Mateo4827c542017-09-07 08:40:07 -07001365 I915_WRITE(GEN7_UCGCTL4, (I915_READ(GEN7_UCGCTL4) |
1366 GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE));
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001367
1368 /* WaDisableSbeCacheDispatchPortSharing:cfl */
1369 WA_SET_BIT_MASKED(
1370 GEN7_HALF_SLICE_CHICKEN1,
1371 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
1372
1373 /* WaInPlaceDecompressionHang:cfl */
Oscar Mateoefc886c2017-09-07 08:40:04 -07001374 I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
1375 (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
1376 GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001377
1378 return 0;
1379}
1380
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001381int init_workarounds_ring(struct intel_engine_cs *engine)
1382{
1383 struct drm_i915_private *dev_priv = engine->i915;
Chris Wilson02e012f2017-03-01 12:11:31 +00001384 int err;
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001385
1386 WARN_ON(engine->id != RCS);
1387
1388 dev_priv->workarounds.count = 0;
Chris Wilson02e012f2017-03-01 12:11:31 +00001389 dev_priv->workarounds.hw_whitelist_count[engine->id] = 0;
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001390
1391 if (IS_BROADWELL(dev_priv))
Chris Wilson02e012f2017-03-01 12:11:31 +00001392 err = bdw_init_workarounds(engine);
1393 else if (IS_CHERRYVIEW(dev_priv))
1394 err = chv_init_workarounds(engine);
1395 else if (IS_SKYLAKE(dev_priv))
1396 err = skl_init_workarounds(engine);
1397 else if (IS_BROXTON(dev_priv))
1398 err = bxt_init_workarounds(engine);
1399 else if (IS_KABYLAKE(dev_priv))
1400 err = kbl_init_workarounds(engine);
1401 else if (IS_GEMINILAKE(dev_priv))
1402 err = glk_init_workarounds(engine);
Rodrigo Vivi46c26662017-06-16 15:49:58 -07001403 else if (IS_COFFEELAKE(dev_priv))
1404 err = cfl_init_workarounds(engine);
Rodrigo Vivi90007bc2017-08-15 16:16:48 -07001405 else if (IS_CANNONLAKE(dev_priv))
1406 err = cnl_init_workarounds(engine);
Chris Wilson02e012f2017-03-01 12:11:31 +00001407 else
1408 err = 0;
1409 if (err)
1410 return err;
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001411
Chris Wilson02e012f2017-03-01 12:11:31 +00001412 DRM_DEBUG_DRIVER("%s: Number of context specific w/a: %d\n",
1413 engine->name, dev_priv->workarounds.count);
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001414 return 0;
1415}
1416
1417int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
1418{
1419 struct i915_workarounds *w = &req->i915->workarounds;
1420 u32 *cs;
1421 int ret, i;
1422
1423 if (w->count == 0)
1424 return 0;
1425
1426 ret = req->engine->emit_flush(req, EMIT_BARRIER);
1427 if (ret)
1428 return ret;
1429
1430 cs = intel_ring_begin(req, (w->count * 2 + 2));
1431 if (IS_ERR(cs))
1432 return PTR_ERR(cs);
1433
1434 *cs++ = MI_LOAD_REGISTER_IMM(w->count);
1435 for (i = 0; i < w->count; i++) {
1436 *cs++ = i915_mmio_reg_offset(w->reg[i].addr);
1437 *cs++ = w->reg[i].value;
1438 }
1439 *cs++ = MI_NOOP;
1440
1441 intel_ring_advance(req, cs);
1442
1443 ret = req->engine->emit_flush(req, EMIT_BARRIER);
1444 if (ret)
1445 return ret;
1446
Tvrtko Ursulin133b4bd2017-02-16 12:23:23 +00001447 return 0;
1448}
1449
Chris Wilsona091d4e2017-05-30 13:13:33 +01001450static bool ring_is_idle(struct intel_engine_cs *engine)
1451{
1452 struct drm_i915_private *dev_priv = engine->i915;
1453 bool idle = true;
1454
1455 intel_runtime_pm_get(dev_priv);
1456
Chris Wilsonaed2fc12017-05-30 13:13:34 +01001457 /* First check that no commands are left in the ring */
1458 if ((I915_READ_HEAD(engine) & HEAD_ADDR) !=
1459 (I915_READ_TAIL(engine) & TAIL_ADDR))
1460 idle = false;
1461
Chris Wilsona091d4e2017-05-30 13:13:33 +01001462 /* No bit for gen2, so assume the CS parser is idle */
1463 if (INTEL_GEN(dev_priv) > 2 && !(I915_READ_MODE(engine) & MODE_IDLE))
1464 idle = false;
1465
1466 intel_runtime_pm_put(dev_priv);
1467
1468 return idle;
1469}
1470
Chris Wilson54003672017-03-03 12:19:46 +00001471/**
1472 * intel_engine_is_idle() - Report if the engine has finished process all work
1473 * @engine: the intel_engine_cs
1474 *
1475 * Return true if there are no requests pending, nothing left to be submitted
1476 * to hardware, and that the engine is idle.
1477 */
1478bool intel_engine_is_idle(struct intel_engine_cs *engine)
1479{
1480 struct drm_i915_private *dev_priv = engine->i915;
1481
Chris Wilsona8e9a412017-04-11 20:00:42 +01001482 /* More white lies, if wedged, hw state is inconsistent */
1483 if (i915_terminally_wedged(&dev_priv->gpu_error))
1484 return true;
1485
Chris Wilson54003672017-03-03 12:19:46 +00001486 /* Any inflight/incomplete requests? */
1487 if (!i915_seqno_passed(intel_engine_get_seqno(engine),
1488 intel_engine_last_submit(engine)))
1489 return false;
1490
Chris Wilson8968a362017-04-12 00:44:26 +01001491 if (I915_SELFTEST_ONLY(engine->breadcrumbs.mock))
1492 return true;
1493
Chris Wilson54003672017-03-03 12:19:46 +00001494 /* Interrupt/tasklet pending? */
1495 if (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted))
1496 return false;
1497
Chris Wilson4a118ec2017-10-23 22:32:36 +01001498 /* Waiting to drain ELSP? */
1499 if (READ_ONCE(engine->execlists.active))
Chris Wilson54003672017-03-03 12:19:46 +00001500 return false;
1501
Chris Wilsond6edb6e2017-07-21 13:32:24 +01001502 /* ELSP is empty, but there are ready requests? */
Mika Kuoppalab620e872017-09-22 15:43:03 +03001503 if (READ_ONCE(engine->execlists.first))
Chris Wilsond6edb6e2017-07-21 13:32:24 +01001504 return false;
1505
Chris Wilson54003672017-03-03 12:19:46 +00001506 /* Ring stopped? */
Chris Wilsona091d4e2017-05-30 13:13:33 +01001507 if (!ring_is_idle(engine))
Chris Wilson54003672017-03-03 12:19:46 +00001508 return false;
1509
1510 return true;
1511}
1512
Chris Wilson05425242017-03-03 12:19:47 +00001513bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
1514{
1515 struct intel_engine_cs *engine;
1516 enum intel_engine_id id;
1517
Chris Wilson8490ae202017-03-30 15:50:37 +01001518 if (READ_ONCE(dev_priv->gt.active_requests))
1519 return false;
1520
1521 /* If the driver is wedged, HW state may be very inconsistent and
1522 * report that it is still busy, even though we have stopped using it.
1523 */
1524 if (i915_terminally_wedged(&dev_priv->gpu_error))
1525 return true;
1526
Chris Wilson05425242017-03-03 12:19:47 +00001527 for_each_engine(engine, dev_priv, id) {
1528 if (!intel_engine_is_idle(engine))
1529 return false;
1530 }
1531
1532 return true;
1533}
1534
Chris Wilsonae6c4572017-11-10 14:26:28 +00001535/**
1536 * intel_engine_has_kernel_context:
1537 * @engine: the engine
1538 *
1539 * Returns true if the last context to be executed on this engine, or has been
1540 * executed if the engine is already idle, is the kernel context
1541 * (#i915.kernel_context).
1542 */
Chris Wilson20ccd4d2017-10-24 23:08:55 +01001543bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine)
1544{
Chris Wilsonae6c4572017-11-10 14:26:28 +00001545 const struct i915_gem_context * const kernel_context =
1546 engine->i915->kernel_context;
1547 struct drm_i915_gem_request *rq;
1548
1549 lockdep_assert_held(&engine->i915->drm.struct_mutex);
1550
1551 /*
1552 * Check the last context seen by the engine. If active, it will be
1553 * the last request that remains in the timeline. When idle, it is
1554 * the last executed context as tracked by retirement.
1555 */
1556 rq = __i915_gem_active_peek(&engine->timeline->last_request);
1557 if (rq)
1558 return rq->ctx == kernel_context;
1559 else
1560 return engine->last_retired_context == kernel_context;
Chris Wilson20ccd4d2017-10-24 23:08:55 +01001561}
1562
Chris Wilsonff44ad52017-03-16 17:13:03 +00001563void intel_engines_reset_default_submission(struct drm_i915_private *i915)
1564{
1565 struct intel_engine_cs *engine;
1566 enum intel_engine_id id;
1567
1568 for_each_engine(engine, i915, id)
1569 engine->set_default_submission(engine);
1570}
1571
Chris Wilsonaba5e272017-10-25 15:39:41 +01001572/**
1573 * intel_engines_park: called when the GT is transitioning from busy->idle
1574 * @i915: the i915 device
1575 *
1576 * The GT is now idle and about to go to sleep (maybe never to wake again?).
1577 * Time for us to tidy and put away our toys (release resources back to the
1578 * system).
1579 */
1580void intel_engines_park(struct drm_i915_private *i915)
Chris Wilson6c067572017-05-17 13:10:03 +01001581{
1582 struct intel_engine_cs *engine;
1583 enum intel_engine_id id;
1584
1585 for_each_engine(engine, i915, id) {
Chris Wilson820c5bb2017-11-01 20:21:49 +00001586 /* Flush the residual irq tasklets first. */
1587 intel_engine_disarm_breadcrumbs(engine);
1588 tasklet_kill(&engine->execlists.irq_tasklet);
1589
Chris Wilson32651242017-10-27 12:06:17 +01001590 /*
1591 * We are committed now to parking the engines, make sure there
1592 * will be no more interrupts arriving later and the engines
1593 * are truly idle.
1594 */
Chris Wilson30b29402017-11-10 11:25:50 +00001595 if (wait_for(intel_engine_is_idle(engine), 10)) {
Chris Wilson32651242017-10-27 12:06:17 +01001596 struct drm_printer p = drm_debug_printer(__func__);
1597
Chris Wilson30b29402017-11-10 11:25:50 +00001598 dev_err(i915->drm.dev,
1599 "%s is not idle before parking\n",
1600 engine->name);
Chris Wilson32651242017-10-27 12:06:17 +01001601 intel_engine_dump(engine, &p);
1602 }
1603
Chris Wilsonaba5e272017-10-25 15:39:41 +01001604 if (engine->park)
1605 engine->park(engine);
1606
Chris Wilsonaba5e272017-10-25 15:39:41 +01001607 i915_gem_batch_pool_fini(&engine->batch_pool);
Mika Kuoppalab620e872017-09-22 15:43:03 +03001608 engine->execlists.no_priolist = false;
Chris Wilson6c067572017-05-17 13:10:03 +01001609 }
1610}
1611
Chris Wilsonaba5e272017-10-25 15:39:41 +01001612/**
1613 * intel_engines_unpark: called when the GT is transitioning from idle->busy
1614 * @i915: the i915 device
1615 *
1616 * The GT was idle and now about to fire up with some new user requests.
1617 */
1618void intel_engines_unpark(struct drm_i915_private *i915)
1619{
1620 struct intel_engine_cs *engine;
1621 enum intel_engine_id id;
1622
1623 for_each_engine(engine, i915, id) {
1624 if (engine->unpark)
1625 engine->unpark(engine);
1626 }
1627}
1628
Chris Wilson90cad092017-09-06 16:28:59 +01001629bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
1630{
1631 switch (INTEL_GEN(engine->i915)) {
1632 case 2:
1633 return false; /* uses physical not virtual addresses */
1634 case 3:
1635 /* maybe only uses physical not virtual addresses */
1636 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915));
1637 case 6:
1638 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */
1639 default:
1640 return true;
1641 }
1642}
1643
Chris Wilsond2b4b972017-11-10 14:26:33 +00001644unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915)
1645{
1646 struct intel_engine_cs *engine;
1647 enum intel_engine_id id;
1648 unsigned int which;
1649
1650 which = 0;
1651 for_each_engine(engine, i915, id)
1652 if (engine->default_state)
1653 which |= BIT(engine->uabi_class);
1654
1655 return which;
1656}
1657
Chris Wilsonf636edb2017-10-09 12:02:57 +01001658static void print_request(struct drm_printer *m,
1659 struct drm_i915_gem_request *rq,
1660 const char *prefix)
1661{
Chris Wilsona27d5a42017-10-15 21:43:10 +01001662 drm_printf(m, "%s%x%s [%x:%x] prio=%d @ %dms: %s\n", prefix,
1663 rq->global_seqno,
1664 i915_gem_request_completed(rq) ? "!" : "",
1665 rq->ctx->hw_id, rq->fence.seqno,
Chris Wilsonf636edb2017-10-09 12:02:57 +01001666 rq->priotree.priority,
1667 jiffies_to_msecs(jiffies - rq->emitted_jiffies),
1668 rq->timeline->common->name);
1669}
1670
1671void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m)
1672{
Chris Wilsona27d5a42017-10-15 21:43:10 +01001673 struct intel_breadcrumbs * const b = &engine->breadcrumbs;
1674 const struct intel_engine_execlists * const execlists = &engine->execlists;
1675 struct i915_gpu_error * const error = &engine->i915->gpu_error;
Chris Wilsonf636edb2017-10-09 12:02:57 +01001676 struct drm_i915_private *dev_priv = engine->i915;
1677 struct drm_i915_gem_request *rq;
1678 struct rb_node *rb;
1679 u64 addr;
1680
1681 drm_printf(m, "%s\n", engine->name);
1682 drm_printf(m, "\tcurrent seqno %x, last %x, hangcheck %x [%d ms], inflight %d\n",
1683 intel_engine_get_seqno(engine),
1684 intel_engine_last_submit(engine),
1685 engine->hangcheck.seqno,
1686 jiffies_to_msecs(jiffies - engine->hangcheck.action_timestamp),
1687 engine->timeline->inflight_seqnos);
1688 drm_printf(m, "\tReset count: %d\n",
1689 i915_reset_engine_count(error, engine));
1690
1691 rcu_read_lock();
1692
1693 drm_printf(m, "\tRequests:\n");
1694
1695 rq = list_first_entry(&engine->timeline->requests,
1696 struct drm_i915_gem_request, link);
1697 if (&rq->link != &engine->timeline->requests)
1698 print_request(m, rq, "\t\tfirst ");
1699
1700 rq = list_last_entry(&engine->timeline->requests,
1701 struct drm_i915_gem_request, link);
1702 if (&rq->link != &engine->timeline->requests)
1703 print_request(m, rq, "\t\tlast ");
1704
1705 rq = i915_gem_find_active_request(engine);
1706 if (rq) {
1707 print_request(m, rq, "\t\tactive ");
1708 drm_printf(m,
1709 "\t\t[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]\n",
1710 rq->head, rq->postfix, rq->tail,
1711 rq->batch ? upper_32_bits(rq->batch->node.start) : ~0u,
1712 rq->batch ? lower_32_bits(rq->batch->node.start) : ~0u);
1713 }
1714
1715 drm_printf(m, "\tRING_START: 0x%08x [0x%08x]\n",
1716 I915_READ(RING_START(engine->mmio_base)),
1717 rq ? i915_ggtt_offset(rq->ring->vma) : 0);
1718 drm_printf(m, "\tRING_HEAD: 0x%08x [0x%08x]\n",
1719 I915_READ(RING_HEAD(engine->mmio_base)) & HEAD_ADDR,
1720 rq ? rq->ring->head : 0);
1721 drm_printf(m, "\tRING_TAIL: 0x%08x [0x%08x]\n",
1722 I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR,
1723 rq ? rq->ring->tail : 0);
Chris Wilson3c75de52017-10-26 12:50:48 +01001724 drm_printf(m, "\tRING_CTL: 0x%08x%s\n",
Chris Wilsonf636edb2017-10-09 12:02:57 +01001725 I915_READ(RING_CTL(engine->mmio_base)),
Chris Wilson3c75de52017-10-26 12:50:48 +01001726 I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
1727 if (INTEL_GEN(engine->i915) > 2) {
1728 drm_printf(m, "\tRING_MODE: 0x%08x%s\n",
1729 I915_READ(RING_MI_MODE(engine->mmio_base)),
1730 I915_READ(RING_MI_MODE(engine->mmio_base)) & (MODE_IDLE) ? " [idle]" : "");
1731 }
Chris Wilsonf636edb2017-10-09 12:02:57 +01001732
1733 rcu_read_unlock();
1734
1735 addr = intel_engine_get_active_head(engine);
1736 drm_printf(m, "\tACTHD: 0x%08x_%08x\n",
1737 upper_32_bits(addr), lower_32_bits(addr));
1738 addr = intel_engine_get_last_batch_head(engine);
1739 drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
1740 upper_32_bits(addr), lower_32_bits(addr));
1741
1742 if (i915_modparams.enable_execlists) {
1743 const u32 *hws = &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
Chris Wilsonf636edb2017-10-09 12:02:57 +01001744 u32 ptr, read, write;
1745 unsigned int idx;
1746
1747 drm_printf(m, "\tExeclist status: 0x%08x %08x\n",
1748 I915_READ(RING_EXECLIST_STATUS_LO(engine)),
1749 I915_READ(RING_EXECLIST_STATUS_HI(engine)));
1750
1751 ptr = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
1752 read = GEN8_CSB_READ_PTR(ptr);
1753 write = GEN8_CSB_WRITE_PTR(ptr);
1754 drm_printf(m, "\tExeclist CSB read %d [%d cached], write %d [%d from hws], interrupt posted? %s\n",
1755 read, execlists->csb_head,
1756 write,
1757 intel_read_status_page(engine, intel_hws_csb_write_index(engine->i915)),
1758 yesno(test_bit(ENGINE_IRQ_EXECLIST,
1759 &engine->irq_posted)));
1760 if (read >= GEN8_CSB_ENTRIES)
1761 read = 0;
1762 if (write >= GEN8_CSB_ENTRIES)
1763 write = 0;
1764 if (read > write)
1765 write += GEN8_CSB_ENTRIES;
1766 while (read < write) {
1767 idx = ++read % GEN8_CSB_ENTRIES;
1768 drm_printf(m, "\tExeclist CSB[%d]: 0x%08x [0x%08x in hwsp], context: %d [%d in hwsp]\n",
1769 idx,
1770 I915_READ(RING_CONTEXT_STATUS_BUF_LO(engine, idx)),
1771 hws[idx * 2],
1772 I915_READ(RING_CONTEXT_STATUS_BUF_HI(engine, idx)),
1773 hws[idx * 2 + 1]);
1774 }
1775
1776 rcu_read_lock();
1777 for (idx = 0; idx < execlists_num_ports(execlists); idx++) {
1778 unsigned int count;
1779
1780 rq = port_unpack(&execlists->port[idx], &count);
1781 if (rq) {
1782 drm_printf(m, "\t\tELSP[%d] count=%d, ",
1783 idx, count);
1784 print_request(m, rq, "rq: ");
1785 } else {
1786 drm_printf(m, "\t\tELSP[%d] idle\n",
1787 idx);
1788 }
1789 }
Chris Wilson4a118ec2017-10-23 22:32:36 +01001790 drm_printf(m, "\t\tHW active? 0x%x\n", execlists->active);
Chris Wilsonf636edb2017-10-09 12:02:57 +01001791 rcu_read_unlock();
Chris Wilsonf636edb2017-10-09 12:02:57 +01001792 } else if (INTEL_GEN(dev_priv) > 6) {
1793 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
1794 I915_READ(RING_PP_DIR_BASE(engine)));
1795 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
1796 I915_READ(RING_PP_DIR_BASE_READ(engine)));
1797 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
1798 I915_READ(RING_PP_DIR_DCLV(engine)));
1799 }
1800
Chris Wilsona27d5a42017-10-15 21:43:10 +01001801 spin_lock_irq(&engine->timeline->lock);
1802 list_for_each_entry(rq, &engine->timeline->requests, link)
1803 print_request(m, rq, "\t\tE ");
1804 for (rb = execlists->first; rb; rb = rb_next(rb)) {
1805 struct i915_priolist *p =
1806 rb_entry(rb, typeof(*p), node);
1807
1808 list_for_each_entry(rq, &p->requests, priotree.link)
1809 print_request(m, rq, "\t\tQ ");
1810 }
1811 spin_unlock_irq(&engine->timeline->lock);
1812
Chris Wilsonf636edb2017-10-09 12:02:57 +01001813 spin_lock_irq(&b->rb_lock);
1814 for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
1815 struct intel_wait *w = rb_entry(rb, typeof(*w), node);
1816
1817 drm_printf(m, "\t%s [%d] waiting for %x\n",
1818 w->tsk->comm, w->tsk->pid, w->seqno);
1819 }
1820 spin_unlock_irq(&b->rb_lock);
1821
Chris Wilsonc400cc22017-11-07 15:22:11 +00001822 drm_printf(m, "Idle? %s\n", yesno(intel_engine_is_idle(engine)));
Chris Wilsonf636edb2017-10-09 12:02:57 +01001823 drm_printf(m, "\n");
1824}
1825
Chris Wilsonf97fbf92017-02-13 17:15:14 +00001826#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1827#include "selftests/mock_engine.c"
1828#endif