blob: 56b30534176aaceebfd867286bd5a3fd2e29f9a3 [file] [log] [blame]
Dave Airlie0d6aa602006-01-02 20:14:23 +11001/* i915_irq.c -- IRQ support for the I915 -*- linux-c -*-
Linus Torvalds1da177e2005-04-16 15:20:36 -07002 */
Dave Airlie0d6aa602006-01-02 20:14:23 +11003/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07004 * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
5 * All Rights Reserved.
Dave Airliebc54fd12005-06-23 22:46:46 +10006 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
Dave Airlie0d6aa602006-01-02 20:14:23 +110027 */
Linus Torvalds1da177e2005-04-16 15:20:36 -070028
Joe Perchesa70491c2012-03-18 13:00:11 -070029#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
Jesse Barnes63eeaf32009-06-18 16:56:52 -070031#include <linux/sysrq.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090032#include <linux/slab.h>
Damien Lespiaub2c88f52013-10-15 18:55:29 +010033#include <linux/circ_buf.h>
David Howells760285e2012-10-02 18:01:07 +010034#include <drm/drmP.h>
35#include <drm/i915_drm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036#include "i915_drv.h"
Chris Wilson1c5d22f2009-08-25 11:15:50 +010037#include "i915_trace.h"
Jesse Barnes79e53942008-11-07 14:24:08 -080038#include "intel_drv.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070039
Daniel Vetterfca52a52014-09-30 10:56:45 +020040/**
41 * DOC: interrupt handling
42 *
43 * These functions provide the basic support for enabling and disabling the
44 * interrupt handling support. There's a lot more functionality in i915_irq.c
45 * and related files, but that will be described in separate chapters.
46 */
47
Egbert Eiche5868a32013-02-28 04:17:12 -050048static const u32 hpd_ibx[] = {
49 [HPD_CRT] = SDE_CRT_HOTPLUG,
50 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG,
51 [HPD_PORT_B] = SDE_PORTB_HOTPLUG,
52 [HPD_PORT_C] = SDE_PORTC_HOTPLUG,
53 [HPD_PORT_D] = SDE_PORTD_HOTPLUG
54};
55
56static const u32 hpd_cpt[] = {
57 [HPD_CRT] = SDE_CRT_HOTPLUG_CPT,
Daniel Vetter73c352a2013-03-26 22:38:43 +010058 [HPD_SDVO_B] = SDE_SDVOB_HOTPLUG_CPT,
Egbert Eiche5868a32013-02-28 04:17:12 -050059 [HPD_PORT_B] = SDE_PORTB_HOTPLUG_CPT,
60 [HPD_PORT_C] = SDE_PORTC_HOTPLUG_CPT,
61 [HPD_PORT_D] = SDE_PORTD_HOTPLUG_CPT
62};
63
64static const u32 hpd_mask_i915[] = {
65 [HPD_CRT] = CRT_HOTPLUG_INT_EN,
66 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_EN,
67 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_EN,
68 [HPD_PORT_B] = PORTB_HOTPLUG_INT_EN,
69 [HPD_PORT_C] = PORTC_HOTPLUG_INT_EN,
70 [HPD_PORT_D] = PORTD_HOTPLUG_INT_EN
71};
72
Daniel Vetter704cfb82013-12-18 09:08:43 +010073static const u32 hpd_status_g4x[] = {
Egbert Eiche5868a32013-02-28 04:17:12 -050074 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
75 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_G4X,
76 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_G4X,
77 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
78 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
79 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
80};
81
Egbert Eiche5868a32013-02-28 04:17:12 -050082static const u32 hpd_status_i915[] = { /* i915 and valleyview are the same */
83 [HPD_CRT] = CRT_HOTPLUG_INT_STATUS,
84 [HPD_SDVO_B] = SDVOB_HOTPLUG_INT_STATUS_I915,
85 [HPD_SDVO_C] = SDVOC_HOTPLUG_INT_STATUS_I915,
86 [HPD_PORT_B] = PORTB_HOTPLUG_INT_STATUS,
87 [HPD_PORT_C] = PORTC_HOTPLUG_INT_STATUS,
88 [HPD_PORT_D] = PORTD_HOTPLUG_INT_STATUS
89};
90
Paulo Zanoni5c502442014-04-01 15:37:11 -030091/* IIR can theoretically queue up two events. Be paranoid. */
Paulo Zanonif86f3fb2014-04-01 15:37:14 -030092#define GEN8_IRQ_RESET_NDX(type, which) do { \
Paulo Zanoni5c502442014-04-01 15:37:11 -030093 I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
94 POSTING_READ(GEN8_##type##_IMR(which)); \
95 I915_WRITE(GEN8_##type##_IER(which), 0); \
96 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
97 POSTING_READ(GEN8_##type##_IIR(which)); \
98 I915_WRITE(GEN8_##type##_IIR(which), 0xffffffff); \
99 POSTING_READ(GEN8_##type##_IIR(which)); \
100} while (0)
101
Paulo Zanonif86f3fb2014-04-01 15:37:14 -0300102#define GEN5_IRQ_RESET(type) do { \
Paulo Zanonia9d356a2014-04-01 15:37:09 -0300103 I915_WRITE(type##IMR, 0xffffffff); \
Paulo Zanoni5c502442014-04-01 15:37:11 -0300104 POSTING_READ(type##IMR); \
Paulo Zanonia9d356a2014-04-01 15:37:09 -0300105 I915_WRITE(type##IER, 0); \
Paulo Zanoni5c502442014-04-01 15:37:11 -0300106 I915_WRITE(type##IIR, 0xffffffff); \
107 POSTING_READ(type##IIR); \
108 I915_WRITE(type##IIR, 0xffffffff); \
109 POSTING_READ(type##IIR); \
Paulo Zanonia9d356a2014-04-01 15:37:09 -0300110} while (0)
111
Paulo Zanoni337ba012014-04-01 15:37:16 -0300112/*
113 * We should clear IMR at preinstall/uninstall, and just check at postinstall.
114 */
115#define GEN5_ASSERT_IIR_IS_ZERO(reg) do { \
116 u32 val = I915_READ(reg); \
117 if (val) { \
118 WARN(1, "Interrupt register 0x%x is not zero: 0x%08x\n", \
119 (reg), val); \
120 I915_WRITE((reg), 0xffffffff); \
121 POSTING_READ(reg); \
122 I915_WRITE((reg), 0xffffffff); \
123 POSTING_READ(reg); \
124 } \
125} while (0)
126
Paulo Zanoni35079892014-04-01 15:37:15 -0300127#define GEN8_IRQ_INIT_NDX(type, which, imr_val, ier_val) do { \
Paulo Zanoni337ba012014-04-01 15:37:16 -0300128 GEN5_ASSERT_IIR_IS_ZERO(GEN8_##type##_IIR(which)); \
Paulo Zanoni35079892014-04-01 15:37:15 -0300129 I915_WRITE(GEN8_##type##_IER(which), (ier_val)); \
Ville Syrjälä7d1bd5392014-10-30 19:42:50 +0200130 I915_WRITE(GEN8_##type##_IMR(which), (imr_val)); \
131 POSTING_READ(GEN8_##type##_IMR(which)); \
Paulo Zanoni35079892014-04-01 15:37:15 -0300132} while (0)
133
134#define GEN5_IRQ_INIT(type, imr_val, ier_val) do { \
Paulo Zanoni337ba012014-04-01 15:37:16 -0300135 GEN5_ASSERT_IIR_IS_ZERO(type##IIR); \
Paulo Zanoni35079892014-04-01 15:37:15 -0300136 I915_WRITE(type##IER, (ier_val)); \
Ville Syrjälä7d1bd5392014-10-30 19:42:50 +0200137 I915_WRITE(type##IMR, (imr_val)); \
138 POSTING_READ(type##IMR); \
Paulo Zanoni35079892014-04-01 15:37:15 -0300139} while (0)
140
Imre Deakc9a9a262014-11-05 20:48:37 +0200141static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir);
142
Zhenyu Wang036a4a72009-06-08 14:40:19 +0800143/* For display hotplug interrupt */
Daniel Vetter47339cd2014-09-30 10:56:46 +0200144void
Jani Nikula2d1013d2014-03-31 14:27:17 +0300145ironlake_enable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
Zhenyu Wang036a4a72009-06-08 14:40:19 +0800146{
Daniel Vetter4bc9d432013-06-27 13:44:58 +0200147 assert_spin_locked(&dev_priv->irq_lock);
148
Jesse Barnes9df7575f2014-06-20 09:29:20 -0700149 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
Paulo Zanonic67a4702013-08-19 13:18:09 -0300150 return;
Paulo Zanonic67a4702013-08-19 13:18:09 -0300151
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000152 if ((dev_priv->irq_mask & mask) != 0) {
153 dev_priv->irq_mask &= ~mask;
154 I915_WRITE(DEIMR, dev_priv->irq_mask);
Chris Wilson3143a2b2010-11-16 15:55:10 +0000155 POSTING_READ(DEIMR);
Zhenyu Wang036a4a72009-06-08 14:40:19 +0800156 }
157}
158
Daniel Vetter47339cd2014-09-30 10:56:46 +0200159void
Jani Nikula2d1013d2014-03-31 14:27:17 +0300160ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
Zhenyu Wang036a4a72009-06-08 14:40:19 +0800161{
Daniel Vetter4bc9d432013-06-27 13:44:58 +0200162 assert_spin_locked(&dev_priv->irq_lock);
163
Paulo Zanoni06ffc772014-07-17 17:43:46 -0300164 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
Paulo Zanonic67a4702013-08-19 13:18:09 -0300165 return;
Paulo Zanonic67a4702013-08-19 13:18:09 -0300166
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000167 if ((dev_priv->irq_mask & mask) != mask) {
168 dev_priv->irq_mask |= mask;
169 I915_WRITE(DEIMR, dev_priv->irq_mask);
Chris Wilson3143a2b2010-11-16 15:55:10 +0000170 POSTING_READ(DEIMR);
Zhenyu Wang036a4a72009-06-08 14:40:19 +0800171 }
172}
173
Paulo Zanoni43eaea12013-08-06 18:57:12 -0300174/**
175 * ilk_update_gt_irq - update GTIMR
176 * @dev_priv: driver private
177 * @interrupt_mask: mask of interrupt bits to update
178 * @enabled_irq_mask: mask of interrupt bits to enable
179 */
180static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
181 uint32_t interrupt_mask,
182 uint32_t enabled_irq_mask)
183{
184 assert_spin_locked(&dev_priv->irq_lock);
185
Jesse Barnes9df7575f2014-06-20 09:29:20 -0700186 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
Paulo Zanonic67a4702013-08-19 13:18:09 -0300187 return;
Paulo Zanonic67a4702013-08-19 13:18:09 -0300188
Paulo Zanoni43eaea12013-08-06 18:57:12 -0300189 dev_priv->gt_irq_mask &= ~interrupt_mask;
190 dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
191 I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
192 POSTING_READ(GTIMR);
193}
194
Daniel Vetter480c8032014-07-16 09:49:40 +0200195void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
Paulo Zanoni43eaea12013-08-06 18:57:12 -0300196{
197 ilk_update_gt_irq(dev_priv, mask, mask);
198}
199
Daniel Vetter480c8032014-07-16 09:49:40 +0200200void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
Paulo Zanoni43eaea12013-08-06 18:57:12 -0300201{
202 ilk_update_gt_irq(dev_priv, mask, 0);
203}
204
Imre Deakb900b942014-11-05 20:48:48 +0200205static u32 gen6_pm_iir(struct drm_i915_private *dev_priv)
206{
207 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IIR(2) : GEN6_PMIIR;
208}
209
Imre Deaka72fbc32014-11-05 20:48:31 +0200210static u32 gen6_pm_imr(struct drm_i915_private *dev_priv)
211{
212 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IMR(2) : GEN6_PMIMR;
213}
214
Imre Deakb900b942014-11-05 20:48:48 +0200215static u32 gen6_pm_ier(struct drm_i915_private *dev_priv)
216{
217 return INTEL_INFO(dev_priv)->gen >= 8 ? GEN8_GT_IER(2) : GEN6_PMIER;
218}
219
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300220/**
221 * snb_update_pm_irq - update GEN6_PMIMR
222 * @dev_priv: driver private
223 * @interrupt_mask: mask of interrupt bits to update
224 * @enabled_irq_mask: mask of interrupt bits to enable
225 */
226static void snb_update_pm_irq(struct drm_i915_private *dev_priv,
227 uint32_t interrupt_mask,
228 uint32_t enabled_irq_mask)
229{
Paulo Zanoni605cd252013-08-06 18:57:15 -0300230 uint32_t new_val;
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300231
232 assert_spin_locked(&dev_priv->irq_lock);
233
Jesse Barnes9df7575f2014-06-20 09:29:20 -0700234 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
Paulo Zanonic67a4702013-08-19 13:18:09 -0300235 return;
Paulo Zanonic67a4702013-08-19 13:18:09 -0300236
Paulo Zanoni605cd252013-08-06 18:57:15 -0300237 new_val = dev_priv->pm_irq_mask;
Paulo Zanonif52ecbc2013-08-06 18:57:14 -0300238 new_val &= ~interrupt_mask;
239 new_val |= (~enabled_irq_mask & interrupt_mask);
240
Paulo Zanoni605cd252013-08-06 18:57:15 -0300241 if (new_val != dev_priv->pm_irq_mask) {
242 dev_priv->pm_irq_mask = new_val;
Imre Deaka72fbc32014-11-05 20:48:31 +0200243 I915_WRITE(gen6_pm_imr(dev_priv), dev_priv->pm_irq_mask);
244 POSTING_READ(gen6_pm_imr(dev_priv));
Paulo Zanonif52ecbc2013-08-06 18:57:14 -0300245 }
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300246}
247
Daniel Vetter480c8032014-07-16 09:49:40 +0200248void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300249{
250 snb_update_pm_irq(dev_priv, mask, mask);
251}
252
Daniel Vetter480c8032014-07-16 09:49:40 +0200253void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
Paulo Zanoniedbfdb42013-08-06 18:57:13 -0300254{
255 snb_update_pm_irq(dev_priv, mask, 0);
256}
257
Imre Deak3cc134e2014-11-19 15:30:03 +0200258void gen6_reset_rps_interrupts(struct drm_device *dev)
259{
260 struct drm_i915_private *dev_priv = dev->dev_private;
261 uint32_t reg = gen6_pm_iir(dev_priv);
262
263 spin_lock_irq(&dev_priv->irq_lock);
264 I915_WRITE(reg, dev_priv->pm_rps_events);
265 I915_WRITE(reg, dev_priv->pm_rps_events);
266 POSTING_READ(reg);
267 spin_unlock_irq(&dev_priv->irq_lock);
268}
269
Imre Deakb900b942014-11-05 20:48:48 +0200270void gen6_enable_rps_interrupts(struct drm_device *dev)
271{
272 struct drm_i915_private *dev_priv = dev->dev_private;
273
274 spin_lock_irq(&dev_priv->irq_lock);
275 WARN_ON(dev_priv->rps.pm_iir);
Imre Deak3cc134e2014-11-19 15:30:03 +0200276 WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
Imre Deakb900b942014-11-05 20:48:48 +0200277 gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
Imre Deakb900b942014-11-05 20:48:48 +0200278 spin_unlock_irq(&dev_priv->irq_lock);
279}
280
281void gen6_disable_rps_interrupts(struct drm_device *dev)
282{
283 struct drm_i915_private *dev_priv = dev->dev_private;
284
285 I915_WRITE(GEN6_PMINTRMSK, INTEL_INFO(dev_priv)->gen >= 8 ?
286 ~GEN8_PMINTR_REDIRECT_TO_NON_DISP : ~0);
287 I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) &
288 ~dev_priv->pm_rps_events);
289 /* Complete PM interrupt masking here doesn't race with the rps work
290 * item again unmasking PM interrupts because that is using a different
291 * register (PMIMR) to mask PM interrupts. The only risk is in leaving
292 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
293
294 spin_lock_irq(&dev_priv->irq_lock);
295 dev_priv->rps.pm_iir = 0;
296 spin_unlock_irq(&dev_priv->irq_lock);
297
298 I915_WRITE(gen6_pm_iir(dev_priv), dev_priv->pm_rps_events);
299}
300
Ben Widawsky09610212014-05-15 20:58:08 +0300301/**
Daniel Vetterfee884e2013-07-04 23:35:21 +0200302 * ibx_display_interrupt_update - update SDEIMR
303 * @dev_priv: driver private
304 * @interrupt_mask: mask of interrupt bits to update
305 * @enabled_irq_mask: mask of interrupt bits to enable
306 */
Daniel Vetter47339cd2014-09-30 10:56:46 +0200307void ibx_display_interrupt_update(struct drm_i915_private *dev_priv,
308 uint32_t interrupt_mask,
309 uint32_t enabled_irq_mask)
Daniel Vetterfee884e2013-07-04 23:35:21 +0200310{
311 uint32_t sdeimr = I915_READ(SDEIMR);
312 sdeimr &= ~interrupt_mask;
313 sdeimr |= (~enabled_irq_mask & interrupt_mask);
314
315 assert_spin_locked(&dev_priv->irq_lock);
316
Jesse Barnes9df7575f2014-06-20 09:29:20 -0700317 if (WARN_ON(!intel_irqs_enabled(dev_priv)))
Paulo Zanonic67a4702013-08-19 13:18:09 -0300318 return;
Paulo Zanonic67a4702013-08-19 13:18:09 -0300319
Daniel Vetterfee884e2013-07-04 23:35:21 +0200320 I915_WRITE(SDEIMR, sdeimr);
321 POSTING_READ(SDEIMR);
322}
Paulo Zanoni86642812013-04-12 17:57:57 -0300323
Daniel Vetterb5ea6422014-03-02 21:18:00 +0100324static void
Imre Deak755e9012014-02-10 18:42:47 +0200325__i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
326 u32 enable_mask, u32 status_mask)
Keith Packard7c463582008-11-04 02:03:27 -0800327{
Ville Syrjälä46c06a32013-02-20 21:16:18 +0200328 u32 reg = PIPESTAT(pipe);
Imre Deak755e9012014-02-10 18:42:47 +0200329 u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
Keith Packard7c463582008-11-04 02:03:27 -0800330
Daniel Vetterb79480b2013-06-27 17:52:10 +0200331 assert_spin_locked(&dev_priv->irq_lock);
Daniel Vetterd518ce52014-08-27 10:43:37 +0200332 WARN_ON(!intel_irqs_enabled(dev_priv));
Daniel Vetterb79480b2013-06-27 17:52:10 +0200333
Ville Syrjälä04feced2014-04-03 13:28:33 +0300334 if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
335 status_mask & ~PIPESTAT_INT_STATUS_MASK,
336 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
337 pipe_name(pipe), enable_mask, status_mask))
Imre Deak755e9012014-02-10 18:42:47 +0200338 return;
339
340 if ((pipestat & enable_mask) == enable_mask)
Ville Syrjälä46c06a32013-02-20 21:16:18 +0200341 return;
342
Imre Deak91d181d2014-02-10 18:42:49 +0200343 dev_priv->pipestat_irq_mask[pipe] |= status_mask;
344
Ville Syrjälä46c06a32013-02-20 21:16:18 +0200345 /* Enable the interrupt, clear any pending status */
Imre Deak755e9012014-02-10 18:42:47 +0200346 pipestat |= enable_mask | status_mask;
Ville Syrjälä46c06a32013-02-20 21:16:18 +0200347 I915_WRITE(reg, pipestat);
348 POSTING_READ(reg);
Keith Packard7c463582008-11-04 02:03:27 -0800349}
350
Daniel Vetterb5ea6422014-03-02 21:18:00 +0100351static void
Imre Deak755e9012014-02-10 18:42:47 +0200352__i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
353 u32 enable_mask, u32 status_mask)
Keith Packard7c463582008-11-04 02:03:27 -0800354{
Ville Syrjälä46c06a32013-02-20 21:16:18 +0200355 u32 reg = PIPESTAT(pipe);
Imre Deak755e9012014-02-10 18:42:47 +0200356 u32 pipestat = I915_READ(reg) & PIPESTAT_INT_ENABLE_MASK;
Keith Packard7c463582008-11-04 02:03:27 -0800357
Daniel Vetterb79480b2013-06-27 17:52:10 +0200358 assert_spin_locked(&dev_priv->irq_lock);
Daniel Vetterd518ce52014-08-27 10:43:37 +0200359 WARN_ON(!intel_irqs_enabled(dev_priv));
Daniel Vetterb79480b2013-06-27 17:52:10 +0200360
Ville Syrjälä04feced2014-04-03 13:28:33 +0300361 if (WARN_ONCE(enable_mask & ~PIPESTAT_INT_ENABLE_MASK ||
362 status_mask & ~PIPESTAT_INT_STATUS_MASK,
363 "pipe %c: enable_mask=0x%x, status_mask=0x%x\n",
364 pipe_name(pipe), enable_mask, status_mask))
Ville Syrjälä46c06a32013-02-20 21:16:18 +0200365 return;
366
Imre Deak755e9012014-02-10 18:42:47 +0200367 if ((pipestat & enable_mask) == 0)
368 return;
369
Imre Deak91d181d2014-02-10 18:42:49 +0200370 dev_priv->pipestat_irq_mask[pipe] &= ~status_mask;
371
Imre Deak755e9012014-02-10 18:42:47 +0200372 pipestat &= ~enable_mask;
Ville Syrjälä46c06a32013-02-20 21:16:18 +0200373 I915_WRITE(reg, pipestat);
374 POSTING_READ(reg);
Keith Packard7c463582008-11-04 02:03:27 -0800375}
376
Imre Deak10c59c52014-02-10 18:42:48 +0200377static u32 vlv_get_pipestat_enable_mask(struct drm_device *dev, u32 status_mask)
378{
379 u32 enable_mask = status_mask << 16;
380
381 /*
Ville Syrjälä724a6902014-04-09 13:28:48 +0300382 * On pipe A we don't support the PSR interrupt yet,
383 * on pipe B and C the same bit MBZ.
Imre Deak10c59c52014-02-10 18:42:48 +0200384 */
385 if (WARN_ON_ONCE(status_mask & PIPE_A_PSR_STATUS_VLV))
386 return 0;
Ville Syrjälä724a6902014-04-09 13:28:48 +0300387 /*
388 * On pipe B and C we don't support the PSR interrupt yet, on pipe
389 * A the same bit is for perf counters which we don't use either.
390 */
391 if (WARN_ON_ONCE(status_mask & PIPE_B_PSR_STATUS_VLV))
392 return 0;
Imre Deak10c59c52014-02-10 18:42:48 +0200393
394 enable_mask &= ~(PIPE_FIFO_UNDERRUN_STATUS |
395 SPRITE0_FLIP_DONE_INT_EN_VLV |
396 SPRITE1_FLIP_DONE_INT_EN_VLV);
397 if (status_mask & SPRITE0_FLIP_DONE_INT_STATUS_VLV)
398 enable_mask |= SPRITE0_FLIP_DONE_INT_EN_VLV;
399 if (status_mask & SPRITE1_FLIP_DONE_INT_STATUS_VLV)
400 enable_mask |= SPRITE1_FLIP_DONE_INT_EN_VLV;
401
402 return enable_mask;
403}
404
Imre Deak755e9012014-02-10 18:42:47 +0200405void
406i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
407 u32 status_mask)
408{
409 u32 enable_mask;
410
Imre Deak10c59c52014-02-10 18:42:48 +0200411 if (IS_VALLEYVIEW(dev_priv->dev))
412 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
413 status_mask);
414 else
415 enable_mask = status_mask << 16;
Imre Deak755e9012014-02-10 18:42:47 +0200416 __i915_enable_pipestat(dev_priv, pipe, enable_mask, status_mask);
417}
418
419void
420i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
421 u32 status_mask)
422{
423 u32 enable_mask;
424
Imre Deak10c59c52014-02-10 18:42:48 +0200425 if (IS_VALLEYVIEW(dev_priv->dev))
426 enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
427 status_mask);
428 else
429 enable_mask = status_mask << 16;
Imre Deak755e9012014-02-10 18:42:47 +0200430 __i915_disable_pipestat(dev_priv, pipe, enable_mask, status_mask);
431}
432
=?utf-8?q?Michel_D=C3=A4nzer?=a6b54f32006-10-24 23:37:43 +1000433/**
Jani Nikulaf49e38d2013-04-29 13:02:54 +0300434 * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
Zhao Yakui01c66882009-10-28 05:10:00 +0000435 */
Jani Nikulaf49e38d2013-04-29 13:02:54 +0300436static void i915_enable_asle_pipestat(struct drm_device *dev)
Zhao Yakui01c66882009-10-28 05:10:00 +0000437{
Jani Nikula2d1013d2014-03-31 14:27:17 +0300438 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000439
Jani Nikulaf49e38d2013-04-29 13:02:54 +0300440 if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
441 return;
442
Daniel Vetter13321782014-09-15 14:55:29 +0200443 spin_lock_irq(&dev_priv->irq_lock);
Zhao Yakui01c66882009-10-28 05:10:00 +0000444
Imre Deak755e9012014-02-10 18:42:47 +0200445 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
Jani Nikulaf8987802013-04-29 13:02:53 +0300446 if (INTEL_INFO(dev)->gen >= 4)
Daniel Vetter3b6c42e2013-10-21 18:04:35 +0200447 i915_enable_pipestat(dev_priv, PIPE_A,
Imre Deak755e9012014-02-10 18:42:47 +0200448 PIPE_LEGACY_BLC_EVENT_STATUS);
Chris Wilson1ec14ad2010-12-04 11:30:53 +0000449
Daniel Vetter13321782014-09-15 14:55:29 +0200450 spin_unlock_irq(&dev_priv->irq_lock);
Zhao Yakui01c66882009-10-28 05:10:00 +0000451}
452
453/**
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700454 * i915_pipe_enabled - check if a pipe is enabled
455 * @dev: DRM device
456 * @pipe: pipe to check
457 *
458 * Reading certain registers when the pipe is disabled can hang the chip.
459 * Use this routine to make sure the PLL is running and the pipe is active
460 * before reading such registers if unsure.
461 */
462static int
463i915_pipe_enabled(struct drm_device *dev, int pipe)
464{
Jani Nikula2d1013d2014-03-31 14:27:17 +0300465 struct drm_i915_private *dev_priv = dev->dev_private;
Paulo Zanoni702e7a52012-10-23 18:29:59 -0200466
Daniel Vettera01025a2013-05-22 00:50:23 +0200467 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
468 /* Locking is horribly broken here, but whatever. */
469 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
470 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
Paulo Zanoni71f8ba62013-05-03 12:15:39 -0300471
Daniel Vettera01025a2013-05-22 00:50:23 +0200472 return intel_crtc->active;
473 } else {
474 return I915_READ(PIPECONF(pipe)) & PIPECONF_ENABLE;
475 }
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700476}
477
Ville Syrjäläf75f3742014-05-15 20:20:36 +0300478/*
479 * This timing diagram depicts the video signal in and
480 * around the vertical blanking period.
481 *
482 * Assumptions about the fictitious mode used in this example:
483 * vblank_start >= 3
484 * vsync_start = vblank_start + 1
485 * vsync_end = vblank_start + 2
486 * vtotal = vblank_start + 3
487 *
488 * start of vblank:
489 * latch double buffered registers
490 * increment frame counter (ctg+)
491 * generate start of vblank interrupt (gen4+)
492 * |
493 * | frame start:
494 * | generate frame start interrupt (aka. vblank interrupt) (gmch)
495 * | may be shifted forward 1-3 extra lines via PIPECONF
496 * | |
497 * | | start of vsync:
498 * | | generate vsync interrupt
499 * | | |
500 * ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx___ ___xxxx
501 * . \hs/ . \hs/ \hs/ \hs/ . \hs/
502 * ----va---> <-----------------vb--------------------> <--------va-------------
503 * | | <----vs-----> |
504 * -vbs-----> <---vbs+1---> <---vbs+2---> <-----0-----> <-----1-----> <-----2--- (scanline counter gen2)
505 * -vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2---> <-----0--- (scanline counter gen3+)
506 * -vbs-2---> <---vbs-2---> <---vbs-1---> <---vbs-----> <---vbs+1---> <---vbs+2- (scanline counter hsw+ hdmi)
507 * | | |
508 * last visible pixel first visible pixel
509 * | increment frame counter (gen3/4)
510 * pixel counter = vblank_start * htotal pixel counter = 0 (gen3/4)
511 *
512 * x = horizontal active
513 * _ = horizontal blanking
514 * hs = horizontal sync
515 * va = vertical active
516 * vb = vertical blanking
517 * vs = vertical sync
518 * vbs = vblank_start (number)
519 *
520 * Summary:
521 * - most events happen at the start of horizontal sync
522 * - frame start happens at the start of horizontal blank, 1-4 lines
523 * (depending on PIPECONF settings) after the start of vblank
524 * - gen3/4 pixel and frame counter are synchronized with the start
525 * of horizontal active on the first line of vertical active
526 */
527
Ville Syrjälä4cdb83e2013-10-11 21:52:44 +0300528static u32 i8xx_get_vblank_counter(struct drm_device *dev, int pipe)
529{
530 /* Gen2 doesn't have a hardware frame counter */
531 return 0;
532}
533
Keith Packard42f52ef2008-10-18 19:39:29 -0700534/* Called from drm generic code, passed a 'crtc', which
535 * we use as a pipe index
536 */
Jesse Barnesf71d4af2011-06-28 13:00:41 -0700537static u32 i915_get_vblank_counter(struct drm_device *dev, int pipe)
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700538{
Jani Nikula2d1013d2014-03-31 14:27:17 +0300539 struct drm_i915_private *dev_priv = dev->dev_private;
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700540 unsigned long high_frame;
541 unsigned long low_frame;
Ville Syrjälä0b2a8e02014-04-29 13:35:50 +0300542 u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700543
544 if (!i915_pipe_enabled(dev, pipe)) {
Zhao Yakui44d98a62009-10-09 11:39:40 +0800545 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
Jesse Barnes9db4a9c2011-02-07 12:26:52 -0800546 "pipe %c\n", pipe_name(pipe));
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700547 return 0;
548 }
549
Ville Syrjälä391f75e2013-09-25 19:55:26 +0300550 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
551 struct intel_crtc *intel_crtc =
552 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
553 const struct drm_display_mode *mode =
554 &intel_crtc->config.adjusted_mode;
555
Ville Syrjälä0b2a8e02014-04-29 13:35:50 +0300556 htotal = mode->crtc_htotal;
557 hsync_start = mode->crtc_hsync_start;
558 vbl_start = mode->crtc_vblank_start;
559 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
560 vbl_start = DIV_ROUND_UP(vbl_start, 2);
Ville Syrjälä391f75e2013-09-25 19:55:26 +0300561 } else {
Daniel Vettera2d213d2014-02-07 16:34:05 +0100562 enum transcoder cpu_transcoder = (enum transcoder) pipe;
Ville Syrjälä391f75e2013-09-25 19:55:26 +0300563
564 htotal = ((I915_READ(HTOTAL(cpu_transcoder)) >> 16) & 0x1fff) + 1;
Ville Syrjälä0b2a8e02014-04-29 13:35:50 +0300565 hsync_start = (I915_READ(HSYNC(cpu_transcoder)) & 0x1fff) + 1;
Ville Syrjälä391f75e2013-09-25 19:55:26 +0300566 vbl_start = (I915_READ(VBLANK(cpu_transcoder)) & 0x1fff) + 1;
Ville Syrjälä0b2a8e02014-04-29 13:35:50 +0300567 if ((I915_READ(PIPECONF(cpu_transcoder)) &
568 PIPECONF_INTERLACE_MASK) != PIPECONF_PROGRESSIVE)
569 vbl_start = DIV_ROUND_UP(vbl_start, 2);
Ville Syrjälä391f75e2013-09-25 19:55:26 +0300570 }
571
Ville Syrjälä0b2a8e02014-04-29 13:35:50 +0300572 /* Convert to pixel count */
573 vbl_start *= htotal;
574
575 /* Start of vblank event occurs at start of hsync */
576 vbl_start -= htotal - hsync_start;
577
Jesse Barnes9db4a9c2011-02-07 12:26:52 -0800578 high_frame = PIPEFRAME(pipe);
579 low_frame = PIPEFRAMEPIXEL(pipe);
Chris Wilson5eddb702010-09-11 13:48:45 +0100580
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700581 /*
582 * High & low register fields aren't synchronized, so make sure
583 * we get a low value that's stable across two reads of the high
584 * register.
585 */
586 do {
Chris Wilson5eddb702010-09-11 13:48:45 +0100587 high1 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
Ville Syrjälä391f75e2013-09-25 19:55:26 +0300588 low = I915_READ(low_frame);
Chris Wilson5eddb702010-09-11 13:48:45 +0100589 high2 = I915_READ(high_frame) & PIPE_FRAME_HIGH_MASK;
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700590 } while (high1 != high2);
591
Chris Wilson5eddb702010-09-11 13:48:45 +0100592 high1 >>= PIPE_FRAME_HIGH_SHIFT;
Ville Syrjälä391f75e2013-09-25 19:55:26 +0300593 pixel = low & PIPE_PIXEL_MASK;
Chris Wilson5eddb702010-09-11 13:48:45 +0100594 low >>= PIPE_FRAME_LOW_SHIFT;
Ville Syrjälä391f75e2013-09-25 19:55:26 +0300595
596 /*
597 * The frame counter increments at beginning of active.
598 * Cook up a vblank counter by also checking the pixel
599 * counter against vblank start.
600 */
Ville Syrjäläedc08d02013-11-06 13:56:27 -0200601 return (((high1 << 8) | low) + (pixel >= vbl_start)) & 0xffffff;
Jesse Barnes0a3e67a2008-09-30 12:14:26 -0700602}
603
Jesse Barnesf71d4af2011-06-28 13:00:41 -0700604static u32 gm45_get_vblank_counter(struct drm_device *dev, int pipe)
Jesse Barnes9880b7a2009-02-06 10:22:41 -0800605{
Jani Nikula2d1013d2014-03-31 14:27:17 +0300606 struct drm_i915_private *dev_priv = dev->dev_private;
Jesse Barnes9db4a9c2011-02-07 12:26:52 -0800607 int reg = PIPE_FRMCOUNT_GM45(pipe);
Jesse Barnes9880b7a2009-02-06 10:22:41 -0800608
609 if (!i915_pipe_enabled(dev, pipe)) {
Zhao Yakui44d98a62009-10-09 11:39:40 +0800610 DRM_DEBUG_DRIVER("trying to get vblank count for disabled "
Jesse Barnes9db4a9c2011-02-07 12:26:52 -0800611 "pipe %c\n", pipe_name(pipe));
Jesse Barnes9880b7a2009-02-06 10:22:41 -0800612 return 0;
613 }
614
615 return I915_READ(reg);
616}
617
Mario Kleinerad3543e2013-10-30 05:13:08 +0100618/* raw reads, only for fast reads of display block, no need for forcewake etc. */
619#define __raw_i915_read32(dev_priv__, reg__) readl((dev_priv__)->regs + (reg__))
Mario Kleinerad3543e2013-10-30 05:13:08 +0100620
Ville Syrjäläa225f072014-04-29 13:35:45 +0300621static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
622{
623 struct drm_device *dev = crtc->base.dev;
624 struct drm_i915_private *dev_priv = dev->dev_private;
625 const struct drm_display_mode *mode = &crtc->config.adjusted_mode;
626 enum pipe pipe = crtc->pipe;
Ville Syrjälä80715b22014-05-15 20:23:23 +0300627 int position, vtotal;
Ville Syrjäläa225f072014-04-29 13:35:45 +0300628
Ville Syrjälä80715b22014-05-15 20:23:23 +0300629 vtotal = mode->crtc_vtotal;
Ville Syrjäläa225f072014-04-29 13:35:45 +0300630 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
631 vtotal /= 2;
632
633 if (IS_GEN2(dev))
634 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
635 else
636 position = __raw_i915_read32(dev_priv, PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
637
638 /*
Ville Syrjälä80715b22014-05-15 20:23:23 +0300639 * See update_scanline_offset() for the details on the
640 * scanline_offset adjustment.
Ville Syrjäläa225f072014-04-29 13:35:45 +0300641 */
Ville Syrjälä80715b22014-05-15 20:23:23 +0300642 return (position + crtc->scanline_offset) % vtotal;
Ville Syrjäläa225f072014-04-29 13:35:45 +0300643}
644
Jesse Barnesf71d4af2011-06-28 13:00:41 -0700645static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
Ville Syrjäläabca9e42013-10-28 20:50:48 +0200646 unsigned int flags, int *vpos, int *hpos,
647 ktime_t *stime, ktime_t *etime)
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100648{
Ville Syrjäläc2baf4b2013-09-23 14:48:50 +0300649 struct drm_i915_private *dev_priv = dev->dev_private;
650 struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
651 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
652 const struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
Ville Syrjälä3aa18df2013-10-11 19:10:32 +0300653 int position;
Ville Syrjälä78e8fc62014-04-29 13:35:44 +0300654 int vbl_start, vbl_end, hsync_start, htotal, vtotal;
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100655 bool in_vbl = true;
656 int ret = 0;
Mario Kleinerad3543e2013-10-30 05:13:08 +0100657 unsigned long irqflags;
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100658
Ville Syrjäläc2baf4b2013-09-23 14:48:50 +0300659 if (!intel_crtc->active) {
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100660 DRM_DEBUG_DRIVER("trying to get scanoutpos for disabled "
Jesse Barnes9db4a9c2011-02-07 12:26:52 -0800661 "pipe %c\n", pipe_name(pipe));
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100662 return 0;
663 }
664
Ville Syrjäläc2baf4b2013-09-23 14:48:50 +0300665 htotal = mode->crtc_htotal;
Ville Syrjälä78e8fc62014-04-29 13:35:44 +0300666 hsync_start = mode->crtc_hsync_start;
Ville Syrjäläc2baf4b2013-09-23 14:48:50 +0300667 vtotal = mode->crtc_vtotal;
668 vbl_start = mode->crtc_vblank_start;
669 vbl_end = mode->crtc_vblank_end;
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100670
Ville Syrjäläd31faf62013-10-28 16:31:41 +0200671 if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
672 vbl_start = DIV_ROUND_UP(vbl_start, 2);
673 vbl_end /= 2;
674 vtotal /= 2;
675 }
676
Ville Syrjäläc2baf4b2013-09-23 14:48:50 +0300677 ret |= DRM_SCANOUTPOS_VALID | DRM_SCANOUTPOS_ACCURATE;
678
Mario Kleinerad3543e2013-10-30 05:13:08 +0100679 /*
680 * Lock uncore.lock, as we will do multiple timing critical raw
681 * register reads, potentially with preemption disabled, so the
682 * following code must not block on uncore.lock.
683 */
684 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
Ville Syrjälä78e8fc62014-04-29 13:35:44 +0300685
Mario Kleinerad3543e2013-10-30 05:13:08 +0100686 /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
687
688 /* Get optional system timestamp before query. */
689 if (stime)
690 *stime = ktime_get();
691
Ville Syrjälä7c06b082013-10-11 21:52:43 +0300692 if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100693 /* No obvious pixelcount register. Only query vertical
694 * scanout position from Display scan line register.
695 */
Ville Syrjäläa225f072014-04-29 13:35:45 +0300696 position = __intel_get_crtc_scanline(intel_crtc);
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100697 } else {
698 /* Have access to pixelcount since start of frame.
699 * We can split this into vertical and horizontal
700 * scanout position.
701 */
Mario Kleinerad3543e2013-10-30 05:13:08 +0100702 position = (__raw_i915_read32(dev_priv, PIPEFRAMEPIXEL(pipe)) & PIPE_PIXEL_MASK) >> PIPE_PIXEL_SHIFT;
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100703
Ville Syrjälä3aa18df2013-10-11 19:10:32 +0300704 /* convert to pixel counts */
705 vbl_start *= htotal;
706 vbl_end *= htotal;
707 vtotal *= htotal;
Ville Syrjälä78e8fc62014-04-29 13:35:44 +0300708
709 /*
Ville Syrjälä7e78f1cb2014-04-29 13:35:49 +0300710 * In interlaced modes, the pixel counter counts all pixels,
711 * so one field will have htotal more pixels. In order to avoid
712 * the reported position from jumping backwards when the pixel
713 * counter is beyond the length of the shorter field, just
714 * clamp the position the length of the shorter field. This
715 * matches how the scanline counter based position works since
716 * the scanline counter doesn't count the two half lines.
717 */
718 if (position >= vtotal)
719 position = vtotal - 1;
720
721 /*
Ville Syrjälä78e8fc62014-04-29 13:35:44 +0300722 * Start of vblank interrupt is triggered at start of hsync,
723 * just prior to the first active line of vblank. However we
724 * consider lines to start at the leading edge of horizontal
725 * active. So, should we get here before we've crossed into
726 * the horizontal active of the first line in vblank, we would
727 * not set the DRM_SCANOUTPOS_INVBL flag. In order to fix that,
728 * always add htotal-hsync_start to the current pixel position.
729 */
730 position = (position + htotal - hsync_start) % vtotal;
Ville Syrjälä3aa18df2013-10-11 19:10:32 +0300731 }
732
Mario Kleinerad3543e2013-10-30 05:13:08 +0100733 /* Get optional system timestamp after query. */
734 if (etime)
735 *etime = ktime_get();
736
737 /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
738
739 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
740
Ville Syrjälä3aa18df2013-10-11 19:10:32 +0300741 in_vbl = position >= vbl_start && position < vbl_end;
742
743 /*
744 * While in vblank, position will be negative
745 * counting up towards 0 at vbl_end. And outside
746 * vblank, position will be positive counting
747 * up since vbl_end.
748 */
749 if (position >= vbl_start)
750 position -= vbl_end;
751 else
752 position += vtotal - vbl_end;
753
Ville Syrjälä7c06b082013-10-11 21:52:43 +0300754 if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
Ville Syrjälä3aa18df2013-10-11 19:10:32 +0300755 *vpos = position;
756 *hpos = 0;
757 } else {
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100758 *vpos = position / htotal;
759 *hpos = position - (*vpos * htotal);
760 }
761
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100762 /* In vblank? */
763 if (in_vbl)
Daniel Vetter3d3cbd82014-09-10 17:36:11 +0200764 ret |= DRM_SCANOUTPOS_IN_VBLANK;
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100765
766 return ret;
767}
768
Ville Syrjäläa225f072014-04-29 13:35:45 +0300769int intel_get_crtc_scanline(struct intel_crtc *crtc)
770{
771 struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
772 unsigned long irqflags;
773 int position;
774
775 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
776 position = __intel_get_crtc_scanline(crtc);
777 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
778
779 return position;
780}
781
Jesse Barnesf71d4af2011-06-28 13:00:41 -0700782static int i915_get_vblank_timestamp(struct drm_device *dev, int pipe,
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100783 int *max_error,
784 struct timeval *vblank_time,
785 unsigned flags)
786{
Chris Wilson4041b852011-01-22 10:07:56 +0000787 struct drm_crtc *crtc;
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100788
Ben Widawsky7eb552a2013-03-13 14:05:41 -0700789 if (pipe < 0 || pipe >= INTEL_INFO(dev)->num_pipes) {
Chris Wilson4041b852011-01-22 10:07:56 +0000790 DRM_ERROR("Invalid crtc %d\n", pipe);
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100791 return -EINVAL;
792 }
793
794 /* Get drm_crtc to timestamp: */
Chris Wilson4041b852011-01-22 10:07:56 +0000795 crtc = intel_get_crtc_for_pipe(dev, pipe);
796 if (crtc == NULL) {
797 DRM_ERROR("Invalid crtc %d\n", pipe);
798 return -EINVAL;
799 }
800
801 if (!crtc->enabled) {
802 DRM_DEBUG_KMS("crtc %d is disabled\n", pipe);
803 return -EBUSY;
804 }
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100805
806 /* Helper routine in DRM core does all the work: */
Chris Wilson4041b852011-01-22 10:07:56 +0000807 return drm_calc_vbltimestamp_from_scanoutpos(dev, pipe, max_error,
808 vblank_time, flags,
Ville Syrjälä7da903e2013-10-26 17:57:31 +0300809 crtc,
810 &to_intel_crtc(crtc)->config.adjusted_mode);
Mario Kleiner0af7e4d2010-12-08 04:07:19 +0100811}
812
Jani Nikula67c347f2013-09-17 14:26:34 +0300813static bool intel_hpd_irq_event(struct drm_device *dev,
814 struct drm_connector *connector)
Egbert Eich321a1b32013-04-11 16:00:26 +0200815{
816 enum drm_connector_status old_status;
817
818 WARN_ON(!mutex_is_locked(&dev->mode_config.mutex));
819 old_status = connector->status;
820
821 connector->status = connector->funcs->detect(connector, false);
Jani Nikula67c347f2013-09-17 14:26:34 +0300822 if (old_status == connector->status)
823 return false;
824
825 DRM_DEBUG_KMS("[CONNECTOR:%d:%s] status updated from %s to %s\n",
Egbert Eich321a1b32013-04-11 16:00:26 +0200826 connector->base.id,
Jani Nikulac23cc412014-06-03 14:56:17 +0300827 connector->name,
Jani Nikula67c347f2013-09-17 14:26:34 +0300828 drm_get_connector_status_name(old_status),
829 drm_get_connector_status_name(connector->status));
830
831 return true;
Egbert Eich321a1b32013-04-11 16:00:26 +0200832}
833
Dave Airlie13cf5502014-06-18 11:29:35 +1000834static void i915_digport_work_func(struct work_struct *work)
835{
836 struct drm_i915_private *dev_priv =
837 container_of(work, struct drm_i915_private, dig_port_work);
Dave Airlie13cf5502014-06-18 11:29:35 +1000838 u32 long_port_mask, short_port_mask;
839 struct intel_digital_port *intel_dig_port;
840 int i, ret;
841 u32 old_bits = 0;
842
Daniel Vetter4cb21832014-09-15 14:55:26 +0200843 spin_lock_irq(&dev_priv->irq_lock);
Dave Airlie13cf5502014-06-18 11:29:35 +1000844 long_port_mask = dev_priv->long_hpd_port_mask;
845 dev_priv->long_hpd_port_mask = 0;
846 short_port_mask = dev_priv->short_hpd_port_mask;
847 dev_priv->short_hpd_port_mask = 0;
Daniel Vetter4cb21832014-09-15 14:55:26 +0200848 spin_unlock_irq(&dev_priv->irq_lock);
Dave Airlie13cf5502014-06-18 11:29:35 +1000849
850 for (i = 0; i < I915_MAX_PORTS; i++) {
851 bool valid = false;
852 bool long_hpd = false;
853 intel_dig_port = dev_priv->hpd_irq_port[i];
854 if (!intel_dig_port || !intel_dig_port->hpd_pulse)
855 continue;
856
857 if (long_port_mask & (1 << i)) {
858 valid = true;
859 long_hpd = true;
860 } else if (short_port_mask & (1 << i))
861 valid = true;
862
863 if (valid) {
864 ret = intel_dig_port->hpd_pulse(intel_dig_port, long_hpd);
865 if (ret == true) {
866 /* if we get true fallback to old school hpd */
867 old_bits |= (1 << intel_dig_port->base.hpd_pin);
868 }
869 }
870 }
871
872 if (old_bits) {
Daniel Vetter4cb21832014-09-15 14:55:26 +0200873 spin_lock_irq(&dev_priv->irq_lock);
Dave Airlie13cf5502014-06-18 11:29:35 +1000874 dev_priv->hpd_event_bits |= old_bits;
Daniel Vetter4cb21832014-09-15 14:55:26 +0200875 spin_unlock_irq(&dev_priv->irq_lock);
Dave Airlie13cf5502014-06-18 11:29:35 +1000876 schedule_work(&dev_priv->hotplug_work);
877 }
878}
879
Jesse Barnes5ca58282009-03-31 14:11:15 -0700880/*
881 * Handle hotplug events outside the interrupt handler proper.
882 */
Egbert Eichac4c16c2013-04-16 13:36:58 +0200883#define I915_REENABLE_HOTPLUG_DELAY (2*60*1000)
884
Jesse Barnes5ca58282009-03-31 14:11:15 -0700885static void i915_hotplug_work_func(struct work_struct *work)
886{
Jani Nikula2d1013d2014-03-31 14:27:17 +0300887 struct drm_i915_private *dev_priv =
888 container_of(work, struct drm_i915_private, hotplug_work);
Jesse Barnes5ca58282009-03-31 14:11:15 -0700889 struct drm_device *dev = dev_priv->dev;
Keith Packardc31c4ba2009-05-06 11:48:58 -0700890 struct drm_mode_config *mode_config = &dev->mode_config;
Egbert Eichcd569ae2013-04-16 13:36:57 +0200891 struct intel_connector *intel_connector;
892 struct intel_encoder *intel_encoder;
893 struct drm_connector *connector;
Egbert Eichcd569ae2013-04-16 13:36:57 +0200894 bool hpd_disabled = false;
Egbert Eich321a1b32013-04-11 16:00:26 +0200895 bool changed = false;
Egbert Eich142e2392013-04-11 15:57:57 +0200896 u32 hpd_event_bits;
Jesse Barnes5ca58282009-03-31 14:11:15 -0700897
Keith Packarda65e34c2011-07-25 10:04:56 -0700898 mutex_lock(&mode_config->mutex);
Jesse Barnese67189ab2011-02-11 14:44:51 -0800899 DRM_DEBUG_KMS("running encoder hotplug functions\n");
900
Daniel Vetter4cb21832014-09-15 14:55:26 +0200901 spin_lock_irq(&dev_priv->irq_lock);
Egbert Eich142e2392013-04-11 15:57:57 +0200902
903 hpd_event_bits = dev_priv->hpd_event_bits;
904 dev_priv->hpd_event_bits = 0;
Egbert Eichcd569ae2013-04-16 13:36:57 +0200905 list_for_each_entry(connector, &mode_config->connector_list, head) {
906 intel_connector = to_intel_connector(connector);
Dave Airlie36cd7442014-05-02 13:44:18 +1000907 if (!intel_connector->encoder)
908 continue;
Egbert Eichcd569ae2013-04-16 13:36:57 +0200909 intel_encoder = intel_connector->encoder;
910 if (intel_encoder->hpd_pin > HPD_NONE &&
911 dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_MARK_DISABLED &&
912 connector->polled == DRM_CONNECTOR_POLL_HPD) {
913 DRM_INFO("HPD interrupt storm detected on connector %s: "
914 "switching from hotplug detection to polling\n",
Jani Nikulac23cc412014-06-03 14:56:17 +0300915 connector->name);
Egbert Eichcd569ae2013-04-16 13:36:57 +0200916 dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark = HPD_DISABLED;
917 connector->polled = DRM_CONNECTOR_POLL_CONNECT
918 | DRM_CONNECTOR_POLL_DISCONNECT;
919 hpd_disabled = true;
920 }
Egbert Eich142e2392013-04-11 15:57:57 +0200921 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
922 DRM_DEBUG_KMS("Connector %s (pin %i) received hotplug event.\n",
Jani Nikulac23cc412014-06-03 14:56:17 +0300923 connector->name, intel_encoder->hpd_pin);
Egbert Eich142e2392013-04-11 15:57:57 +0200924 }
Egbert Eichcd569ae2013-04-16 13:36:57 +0200925 }
926 /* if there were no outputs to poll, poll was disabled,
927 * therefore make sure it's enabled when disabling HPD on
928 * some connectors */
Egbert Eichac4c16c2013-04-16 13:36:58 +0200929 if (hpd_disabled) {
Egbert Eichcd569ae2013-04-16 13:36:57 +0200930 drm_kms_helper_poll_enable(dev);
Imre Deak63237512014-08-18 15:37:02 +0300931 mod_delayed_work(system_wq, &dev_priv->hotplug_reenable_work,
932 msecs_to_jiffies(I915_REENABLE_HOTPLUG_DELAY));
Egbert Eichac4c16c2013-04-16 13:36:58 +0200933 }
Egbert Eichcd569ae2013-04-16 13:36:57 +0200934
Daniel Vetter4cb21832014-09-15 14:55:26 +0200935 spin_unlock_irq(&dev_priv->irq_lock);
Egbert Eichcd569ae2013-04-16 13:36:57 +0200936
Egbert Eich321a1b32013-04-11 16:00:26 +0200937 list_for_each_entry(connector, &mode_config->connector_list, head) {
938 intel_connector = to_intel_connector(connector);
Dave Airlie36cd7442014-05-02 13:44:18 +1000939 if (!intel_connector->encoder)
940 continue;
Egbert Eich321a1b32013-04-11 16:00:26 +0200941 intel_encoder = intel_connector->encoder;
942 if (hpd_event_bits & (1 << intel_encoder->hpd_pin)) {
943 if (intel_encoder->hot_plug)
944 intel_encoder->hot_plug(intel_encoder);
945 if (intel_hpd_irq_event(dev, connector))
946 changed = true;
947 }
948 }
Keith Packard40ee3382011-07-28 15:31:19 -0700949 mutex_unlock(&mode_config->mutex);
950
Egbert Eich321a1b32013-04-11 16:00:26 +0200951 if (changed)
952 drm_kms_helper_hotplug_event(dev);
Jesse Barnes5ca58282009-03-31 14:11:15 -0700953}
954
Daniel Vetterd0ecd7e2013-07-04 23:35:25 +0200955static void ironlake_rps_change_irq_handler(struct drm_device *dev)
Jesse Barnesf97108d2010-01-29 11:27:07 -0800956{
Jani Nikula2d1013d2014-03-31 14:27:17 +0300957 struct drm_i915_private *dev_priv = dev->dev_private;
Matthew Garrettb5b72e82010-02-02 18:30:47 +0000958 u32 busy_up, busy_down, max_avg, min_avg;
Daniel Vetter92703882012-08-09 16:46:01 +0200959 u8 new_delay;
Daniel Vetter92703882012-08-09 16:46:01 +0200960
Daniel Vetterd0ecd7e2013-07-04 23:35:25 +0200961 spin_lock(&mchdev_lock);
Jesse Barnesf97108d2010-01-29 11:27:07 -0800962
Daniel Vetter73edd18f2012-08-08 23:35:37 +0200963 I915_WRITE16(MEMINTRSTS, I915_READ(MEMINTRSTS));
964
Daniel Vetter20e4d402012-08-08 23:35:39 +0200965 new_delay = dev_priv->ips.cur_delay;
Daniel Vetter92703882012-08-09 16:46:01 +0200966
Jesse Barnes7648fa92010-05-20 14:28:11 -0700967 I915_WRITE16(MEMINTRSTS, MEMINT_EVAL_CHG);
Matthew Garrettb5b72e82010-02-02 18:30:47 +0000968 busy_up = I915_READ(RCPREVBSYTUPAVG);
969 busy_down = I915_READ(RCPREVBSYTDNAVG);
Jesse Barnesf97108d2010-01-29 11:27:07 -0800970 max_avg = I915_READ(RCBMAXAVG);
971 min_avg = I915_READ(RCBMINAVG);
972
973 /* Handle RCS change request from hw */
Matthew Garrettb5b72e82010-02-02 18:30:47 +0000974 if (busy_up > max_avg) {
Daniel Vetter20e4d402012-08-08 23:35:39 +0200975 if (dev_priv->ips.cur_delay != dev_priv->ips.max_delay)
976 new_delay = dev_priv->ips.cur_delay - 1;
977 if (new_delay < dev_priv->ips.max_delay)
978 new_delay = dev_priv->ips.max_delay;
Matthew Garrettb5b72e82010-02-02 18:30:47 +0000979 } else if (busy_down < min_avg) {
Daniel Vetter20e4d402012-08-08 23:35:39 +0200980 if (dev_priv->ips.cur_delay != dev_priv->ips.min_delay)
981 new_delay = dev_priv->ips.cur_delay + 1;
982 if (new_delay > dev_priv->ips.min_delay)
983 new_delay = dev_priv->ips.min_delay;
Jesse Barnesf97108d2010-01-29 11:27:07 -0800984 }
985
Jesse Barnes7648fa92010-05-20 14:28:11 -0700986 if (ironlake_set_drps(dev, new_delay))
Daniel Vetter20e4d402012-08-08 23:35:39 +0200987 dev_priv->ips.cur_delay = new_delay;
Jesse Barnesf97108d2010-01-29 11:27:07 -0800988
Daniel Vetterd0ecd7e2013-07-04 23:35:25 +0200989 spin_unlock(&mchdev_lock);
Daniel Vetter92703882012-08-09 16:46:01 +0200990
Jesse Barnesf97108d2010-01-29 11:27:07 -0800991 return;
992}
993
Chris Wilson549f7362010-10-19 11:19:32 +0100994static void notify_ring(struct drm_device *dev,
Oscar Mateoa4872ba2014-05-22 14:13:33 +0100995 struct intel_engine_cs *ring)
Chris Wilson549f7362010-10-19 11:19:32 +0100996{
Oscar Mateo93b0a4e2014-05-22 14:13:36 +0100997 if (!intel_ring_initialized(ring))
Chris Wilson475553d2011-01-20 09:52:56 +0000998 return;
999
Chris Wilson814e9b52013-09-23 17:33:19 -03001000 trace_i915_gem_request_complete(ring);
Chris Wilson9862e602011-01-04 22:22:17 +00001001
Chris Wilson549f7362010-10-19 11:19:32 +01001002 wake_up_all(&ring->irq_queue);
Chris Wilson549f7362010-10-19 11:19:32 +01001003}
1004
Deepak S31685c22014-07-03 17:33:01 -04001005static u32 vlv_c0_residency(struct drm_i915_private *dev_priv,
Chris Wilsonbf225f22014-07-10 20:31:18 +01001006 struct intel_rps_ei *rps_ei)
Deepak S31685c22014-07-03 17:33:01 -04001007{
1008 u32 cz_ts, cz_freq_khz;
1009 u32 render_count, media_count;
1010 u32 elapsed_render, elapsed_media, elapsed_time;
1011 u32 residency = 0;
1012
1013 cz_ts = vlv_punit_read(dev_priv, PUNIT_REG_CZ_TIMESTAMP);
1014 cz_freq_khz = DIV_ROUND_CLOSEST(dev_priv->mem_freq * 1000, 4);
1015
1016 render_count = I915_READ(VLV_RENDER_C0_COUNT_REG);
1017 media_count = I915_READ(VLV_MEDIA_C0_COUNT_REG);
1018
Chris Wilsonbf225f22014-07-10 20:31:18 +01001019 if (rps_ei->cz_clock == 0) {
1020 rps_ei->cz_clock = cz_ts;
1021 rps_ei->render_c0 = render_count;
1022 rps_ei->media_c0 = media_count;
Deepak S31685c22014-07-03 17:33:01 -04001023
1024 return dev_priv->rps.cur_freq;
1025 }
1026
Chris Wilsonbf225f22014-07-10 20:31:18 +01001027 elapsed_time = cz_ts - rps_ei->cz_clock;
1028 rps_ei->cz_clock = cz_ts;
Deepak S31685c22014-07-03 17:33:01 -04001029
Chris Wilsonbf225f22014-07-10 20:31:18 +01001030 elapsed_render = render_count - rps_ei->render_c0;
1031 rps_ei->render_c0 = render_count;
Deepak S31685c22014-07-03 17:33:01 -04001032
Chris Wilsonbf225f22014-07-10 20:31:18 +01001033 elapsed_media = media_count - rps_ei->media_c0;
1034 rps_ei->media_c0 = media_count;
Deepak S31685c22014-07-03 17:33:01 -04001035
1036 /* Convert all the counters into common unit of milli sec */
1037 elapsed_time /= VLV_CZ_CLOCK_TO_MILLI_SEC;
1038 elapsed_render /= cz_freq_khz;
1039 elapsed_media /= cz_freq_khz;
1040
1041 /*
1042 * Calculate overall C0 residency percentage
1043 * only if elapsed time is non zero
1044 */
1045 if (elapsed_time) {
1046 residency =
1047 ((max(elapsed_render, elapsed_media) * 100)
1048 / elapsed_time);
1049 }
1050
1051 return residency;
1052}
1053
1054/**
1055 * vlv_calc_delay_from_C0_counters - Increase/Decrease freq based on GPU
1056 * busy-ness calculated from C0 counters of render & media power wells
1057 * @dev_priv: DRM device private
1058 *
1059 */
Damien Lespiau4fa79042014-08-08 19:25:57 +01001060static int vlv_calc_delay_from_C0_counters(struct drm_i915_private *dev_priv)
Deepak S31685c22014-07-03 17:33:01 -04001061{
1062 u32 residency_C0_up = 0, residency_C0_down = 0;
Damien Lespiau4fa79042014-08-08 19:25:57 +01001063 int new_delay, adj;
Deepak S31685c22014-07-03 17:33:01 -04001064
1065 dev_priv->rps.ei_interrupt_count++;
1066
1067 WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
1068
1069
Chris Wilsonbf225f22014-07-10 20:31:18 +01001070 if (dev_priv->rps.up_ei.cz_clock == 0) {
1071 vlv_c0_residency(dev_priv, &dev_priv->rps.up_ei);
1072 vlv_c0_residency(dev_priv, &dev_priv->rps.down_ei);
Deepak S31685c22014-07-03 17:33:01 -04001073 return dev_priv->rps.cur_freq;
1074 }
1075
1076
1077 /*
1078 * To down throttle, C0 residency should be less than down threshold
1079 * for continous EI intervals. So calculate down EI counters
1080 * once in VLV_INT_COUNT_FOR_DOWN_EI
1081 */
1082 if (dev_priv->rps.ei_interrupt_count == VLV_INT_COUNT_FOR_DOWN_EI) {
1083
1084 dev_priv->rps.ei_interrupt_count = 0;
1085
1086 residency_C0_down = vlv_c0_residency(dev_priv,
Chris Wilsonbf225f22014-07-10 20:31:18 +01001087 &dev_priv->rps.down_ei);
Deepak S31685c22014-07-03 17:33:01 -04001088 } else {
1089 residency_C0_up = vlv_c0_residency(dev_priv,
Chris Wilsonbf225f22014-07-10 20:31:18 +01001090 &dev_priv->rps.up_ei);
Deepak S31685c22014-07-03 17:33:01 -04001091 }
1092
1093 new_delay = dev_priv->rps.cur_freq;
1094
1095 adj = dev_priv->rps.last_adj;
1096 /* C0 residency is greater than UP threshold. Increase Frequency */
1097 if (residency_C0_up >= VLV_RP_UP_EI_THRESHOLD) {
1098 if (adj > 0)
1099 adj *= 2;
1100 else
1101 adj = 1;
1102
1103 if (dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit)
1104 new_delay = dev_priv->rps.cur_freq + adj;
1105
1106 /*
1107 * For better performance, jump directly
1108 * to RPe if we're below it.
1109 */
1110 if (new_delay < dev_priv->rps.efficient_freq)
1111 new_delay = dev_priv->rps.efficient_freq;
1112
1113 } else if (!dev_priv->rps.ei_interrupt_count &&
1114 (residency_C0_down < VLV_RP_DOWN_EI_THRESHOLD)) {
1115 if (adj < 0)
1116 adj *= 2;
1117 else
1118 adj = -1;
1119 /*
1120 * This means, C0 residency is less than down threshold over
1121 * a period of VLV_INT_COUNT_FOR_DOWN_EI. So, reduce the freq
1122 */
1123 if (dev_priv->rps.cur_freq > dev_priv->rps.min_freq_softlimit)
1124 new_delay = dev_priv->rps.cur_freq + adj;
1125 }
1126
1127 return new_delay;
1128}
1129
Ben Widawsky4912d042011-04-25 11:25:20 -07001130static void gen6_pm_rps_work(struct work_struct *work)
Jesse Barnes3b8d8d92010-12-17 14:19:02 -08001131{
Jani Nikula2d1013d2014-03-31 14:27:17 +03001132 struct drm_i915_private *dev_priv =
1133 container_of(work, struct drm_i915_private, rps.work);
Paulo Zanoniedbfdb42013-08-06 18:57:13 -03001134 u32 pm_iir;
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001135 int new_delay, adj;
Jesse Barnes3b8d8d92010-12-17 14:19:02 -08001136
Daniel Vetter59cdb632013-07-04 23:35:28 +02001137 spin_lock_irq(&dev_priv->irq_lock);
Daniel Vetterc6a828d2012-08-08 23:35:35 +02001138 pm_iir = dev_priv->rps.pm_iir;
1139 dev_priv->rps.pm_iir = 0;
Imre Deaka72fbc32014-11-05 20:48:31 +02001140 /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
1141 gen6_enable_pm_irq(dev_priv, dev_priv->pm_rps_events);
Daniel Vetter59cdb632013-07-04 23:35:28 +02001142 spin_unlock_irq(&dev_priv->irq_lock);
Ben Widawsky4912d042011-04-25 11:25:20 -07001143
Paulo Zanoni60611c12013-08-15 11:50:01 -03001144 /* Make sure we didn't queue anything we're not going to process. */
Deepak Sa6706b42014-03-15 20:23:22 +05301145 WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
Paulo Zanoni60611c12013-08-15 11:50:01 -03001146
Deepak Sa6706b42014-03-15 20:23:22 +05301147 if ((pm_iir & dev_priv->pm_rps_events) == 0)
Jesse Barnes3b8d8d92010-12-17 14:19:02 -08001148 return;
1149
Jesse Barnes4fc688c2012-11-02 11:14:01 -07001150 mutex_lock(&dev_priv->rps.hw_lock);
Chris Wilson7b9e0ae2012-04-28 08:56:39 +01001151
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001152 adj = dev_priv->rps.last_adj;
Ville Syrjälä74250342013-06-25 21:38:11 +03001153 if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) {
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001154 if (adj > 0)
1155 adj *= 2;
Deepak S13a56602014-05-23 21:00:21 +05301156 else {
1157 /* CHV needs even encode values */
1158 adj = IS_CHERRYVIEW(dev_priv->dev) ? 2 : 1;
1159 }
Ben Widawskyb39fb292014-03-19 18:31:11 -07001160 new_delay = dev_priv->rps.cur_freq + adj;
Ville Syrjälä74250342013-06-25 21:38:11 +03001161
1162 /*
1163 * For better performance, jump directly
1164 * to RPe if we're below it.
1165 */
Ben Widawskyb39fb292014-03-19 18:31:11 -07001166 if (new_delay < dev_priv->rps.efficient_freq)
1167 new_delay = dev_priv->rps.efficient_freq;
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001168 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) {
Ben Widawskyb39fb292014-03-19 18:31:11 -07001169 if (dev_priv->rps.cur_freq > dev_priv->rps.efficient_freq)
1170 new_delay = dev_priv->rps.efficient_freq;
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001171 else
Ben Widawskyb39fb292014-03-19 18:31:11 -07001172 new_delay = dev_priv->rps.min_freq_softlimit;
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001173 adj = 0;
Deepak S31685c22014-07-03 17:33:01 -04001174 } else if (pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) {
1175 new_delay = vlv_calc_delay_from_C0_counters(dev_priv);
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001176 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) {
1177 if (adj < 0)
1178 adj *= 2;
Deepak S13a56602014-05-23 21:00:21 +05301179 else {
1180 /* CHV needs even encode values */
1181 adj = IS_CHERRYVIEW(dev_priv->dev) ? -2 : -1;
1182 }
Ben Widawskyb39fb292014-03-19 18:31:11 -07001183 new_delay = dev_priv->rps.cur_freq + adj;
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001184 } else { /* unknown event */
Ben Widawskyb39fb292014-03-19 18:31:11 -07001185 new_delay = dev_priv->rps.cur_freq;
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001186 }
Jesse Barnes3b8d8d92010-12-17 14:19:02 -08001187
Ben Widawsky79249632012-09-07 19:43:42 -07001188 /* sysfs frequency interfaces may have snuck in while servicing the
1189 * interrupt
1190 */
Ville Syrjälä1272e7b2013-11-07 19:57:49 +02001191 new_delay = clamp_t(int, new_delay,
Ben Widawskyb39fb292014-03-19 18:31:11 -07001192 dev_priv->rps.min_freq_softlimit,
1193 dev_priv->rps.max_freq_softlimit);
Deepak S27544362014-01-27 21:35:05 +05301194
Ben Widawskyb39fb292014-03-19 18:31:11 -07001195 dev_priv->rps.last_adj = new_delay - dev_priv->rps.cur_freq;
Chris Wilsondd75fdc2013-09-25 17:34:57 +01001196
1197 if (IS_VALLEYVIEW(dev_priv->dev))
1198 valleyview_set_rps(dev_priv->dev, new_delay);
1199 else
1200 gen6_set_rps(dev_priv->dev, new_delay);
Jesse Barnes3b8d8d92010-12-17 14:19:02 -08001201
Jesse Barnes4fc688c2012-11-02 11:14:01 -07001202 mutex_unlock(&dev_priv->rps.hw_lock);
Jesse Barnes3b8d8d92010-12-17 14:19:02 -08001203}
1204
Ben Widawskye3689192012-05-25 16:56:22 -07001205
1206/**
1207 * ivybridge_parity_work - Workqueue called when a parity error interrupt
1208 * occurred.
1209 * @work: workqueue struct
1210 *
1211 * Doesn't actually do anything except notify userspace. As a consequence of
1212 * this event, userspace should try to remap the bad rows since statistically
1213 * it is likely the same row is more likely to go bad again.
1214 */
1215static void ivybridge_parity_work(struct work_struct *work)
1216{
Jani Nikula2d1013d2014-03-31 14:27:17 +03001217 struct drm_i915_private *dev_priv =
1218 container_of(work, struct drm_i915_private, l3_parity.error_work);
Ben Widawskye3689192012-05-25 16:56:22 -07001219 u32 error_status, row, bank, subbank;
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001220 char *parity_event[6];
Ben Widawskye3689192012-05-25 16:56:22 -07001221 uint32_t misccpctl;
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001222 uint8_t slice = 0;
Ben Widawskye3689192012-05-25 16:56:22 -07001223
1224 /* We must turn off DOP level clock gating to access the L3 registers.
1225 * In order to prevent a get/put style interface, acquire struct mutex
1226 * any time we access those registers.
1227 */
1228 mutex_lock(&dev_priv->dev->struct_mutex);
1229
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001230 /* If we've screwed up tracking, just let the interrupt fire again */
1231 if (WARN_ON(!dev_priv->l3_parity.which_slice))
1232 goto out;
1233
Ben Widawskye3689192012-05-25 16:56:22 -07001234 misccpctl = I915_READ(GEN7_MISCCPCTL);
1235 I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
1236 POSTING_READ(GEN7_MISCCPCTL);
1237
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001238 while ((slice = ffs(dev_priv->l3_parity.which_slice)) != 0) {
1239 u32 reg;
Ben Widawskye3689192012-05-25 16:56:22 -07001240
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001241 slice--;
1242 if (WARN_ON_ONCE(slice >= NUM_L3_SLICES(dev_priv->dev)))
1243 break;
1244
1245 dev_priv->l3_parity.which_slice &= ~(1<<slice);
1246
1247 reg = GEN7_L3CDERRST1 + (slice * 0x200);
1248
1249 error_status = I915_READ(reg);
1250 row = GEN7_PARITY_ERROR_ROW(error_status);
1251 bank = GEN7_PARITY_ERROR_BANK(error_status);
1252 subbank = GEN7_PARITY_ERROR_SUBBANK(error_status);
1253
1254 I915_WRITE(reg, GEN7_PARITY_ERROR_VALID | GEN7_L3CDERRST1_ENABLE);
1255 POSTING_READ(reg);
1256
1257 parity_event[0] = I915_L3_PARITY_UEVENT "=1";
1258 parity_event[1] = kasprintf(GFP_KERNEL, "ROW=%d", row);
1259 parity_event[2] = kasprintf(GFP_KERNEL, "BANK=%d", bank);
1260 parity_event[3] = kasprintf(GFP_KERNEL, "SUBBANK=%d", subbank);
1261 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
1262 parity_event[5] = NULL;
1263
Dave Airlie5bdebb12013-10-11 14:07:25 +10001264 kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001265 KOBJ_CHANGE, parity_event);
1266
1267 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
1268 slice, row, bank, subbank);
1269
1270 kfree(parity_event[4]);
1271 kfree(parity_event[3]);
1272 kfree(parity_event[2]);
1273 kfree(parity_event[1]);
1274 }
Ben Widawskye3689192012-05-25 16:56:22 -07001275
1276 I915_WRITE(GEN7_MISCCPCTL, misccpctl);
1277
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001278out:
1279 WARN_ON(dev_priv->l3_parity.which_slice);
Daniel Vetter4cb21832014-09-15 14:55:26 +02001280 spin_lock_irq(&dev_priv->irq_lock);
Daniel Vetter480c8032014-07-16 09:49:40 +02001281 gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv->dev));
Daniel Vetter4cb21832014-09-15 14:55:26 +02001282 spin_unlock_irq(&dev_priv->irq_lock);
Ben Widawskye3689192012-05-25 16:56:22 -07001283
1284 mutex_unlock(&dev_priv->dev->struct_mutex);
Ben Widawskye3689192012-05-25 16:56:22 -07001285}
1286
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001287static void ivybridge_parity_error_irq_handler(struct drm_device *dev, u32 iir)
Ben Widawskye3689192012-05-25 16:56:22 -07001288{
Jani Nikula2d1013d2014-03-31 14:27:17 +03001289 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawskye3689192012-05-25 16:56:22 -07001290
Ben Widawsky040d2ba2013-09-19 11:01:40 -07001291 if (!HAS_L3_DPF(dev))
Ben Widawskye3689192012-05-25 16:56:22 -07001292 return;
1293
Daniel Vetterd0ecd7e2013-07-04 23:35:25 +02001294 spin_lock(&dev_priv->irq_lock);
Daniel Vetter480c8032014-07-16 09:49:40 +02001295 gen5_disable_gt_irq(dev_priv, GT_PARITY_ERROR(dev));
Daniel Vetterd0ecd7e2013-07-04 23:35:25 +02001296 spin_unlock(&dev_priv->irq_lock);
Ben Widawskye3689192012-05-25 16:56:22 -07001297
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001298 iir &= GT_PARITY_ERROR(dev);
1299 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1)
1300 dev_priv->l3_parity.which_slice |= 1 << 1;
1301
1302 if (iir & GT_RENDER_L3_PARITY_ERROR_INTERRUPT)
1303 dev_priv->l3_parity.which_slice |= 1 << 0;
1304
Daniel Vettera4da4fa2012-11-02 19:55:07 +01001305 queue_work(dev_priv->wq, &dev_priv->l3_parity.error_work);
Ben Widawskye3689192012-05-25 16:56:22 -07001306}
1307
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03001308static void ilk_gt_irq_handler(struct drm_device *dev,
1309 struct drm_i915_private *dev_priv,
1310 u32 gt_iir)
1311{
1312 if (gt_iir &
1313 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
1314 notify_ring(dev, &dev_priv->ring[RCS]);
1315 if (gt_iir & ILK_BSD_USER_INTERRUPT)
1316 notify_ring(dev, &dev_priv->ring[VCS]);
1317}
1318
Daniel Vettere7b4c6b2012-03-30 20:24:35 +02001319static void snb_gt_irq_handler(struct drm_device *dev,
1320 struct drm_i915_private *dev_priv,
1321 u32 gt_iir)
1322{
1323
Ben Widawskycc609d52013-05-28 19:22:29 -07001324 if (gt_iir &
1325 (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
Daniel Vettere7b4c6b2012-03-30 20:24:35 +02001326 notify_ring(dev, &dev_priv->ring[RCS]);
Ben Widawskycc609d52013-05-28 19:22:29 -07001327 if (gt_iir & GT_BSD_USER_INTERRUPT)
Daniel Vettere7b4c6b2012-03-30 20:24:35 +02001328 notify_ring(dev, &dev_priv->ring[VCS]);
Ben Widawskycc609d52013-05-28 19:22:29 -07001329 if (gt_iir & GT_BLT_USER_INTERRUPT)
Daniel Vettere7b4c6b2012-03-30 20:24:35 +02001330 notify_ring(dev, &dev_priv->ring[BCS]);
1331
Ben Widawskycc609d52013-05-28 19:22:29 -07001332 if (gt_iir & (GT_BLT_CS_ERROR_INTERRUPT |
1333 GT_BSD_CS_ERROR_INTERRUPT |
1334 GT_RENDER_CS_MASTER_ERROR_INTERRUPT)) {
Mika Kuoppala58174462014-02-25 17:11:26 +02001335 i915_handle_error(dev, false, "GT error interrupt 0x%08x",
1336 gt_iir);
Daniel Vettere7b4c6b2012-03-30 20:24:35 +02001337 }
Ben Widawskye3689192012-05-25 16:56:22 -07001338
Ben Widawsky35a85ac2013-09-19 11:13:41 -07001339 if (gt_iir & GT_PARITY_ERROR(dev))
1340 ivybridge_parity_error_irq_handler(dev, gt_iir);
Daniel Vettere7b4c6b2012-03-30 20:24:35 +02001341}
1342
Ben Widawskyabd58f02013-11-02 21:07:09 -07001343static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
1344 struct drm_i915_private *dev_priv,
1345 u32 master_ctl)
1346{
Thomas Daniele981e7b2014-07-24 17:04:39 +01001347 struct intel_engine_cs *ring;
Ben Widawskyabd58f02013-11-02 21:07:09 -07001348 u32 rcs, bcs, vcs;
1349 uint32_t tmp = 0;
1350 irqreturn_t ret = IRQ_NONE;
1351
1352 if (master_ctl & (GEN8_GT_RCS_IRQ | GEN8_GT_BCS_IRQ)) {
1353 tmp = I915_READ(GEN8_GT_IIR(0));
1354 if (tmp) {
Oscar Mateo38cc46d2014-06-16 16:10:59 +01001355 I915_WRITE(GEN8_GT_IIR(0), tmp);
Ben Widawskyabd58f02013-11-02 21:07:09 -07001356 ret = IRQ_HANDLED;
Thomas Daniele981e7b2014-07-24 17:04:39 +01001357
Ben Widawskyabd58f02013-11-02 21:07:09 -07001358 rcs = tmp >> GEN8_RCS_IRQ_SHIFT;
Thomas Daniele981e7b2014-07-24 17:04:39 +01001359 ring = &dev_priv->ring[RCS];
Ben Widawskyabd58f02013-11-02 21:07:09 -07001360 if (rcs & GT_RENDER_USER_INTERRUPT)
Thomas Daniele981e7b2014-07-24 17:04:39 +01001361 notify_ring(dev, ring);
1362 if (rcs & GT_CONTEXT_SWITCH_INTERRUPT)
1363 intel_execlists_handle_ctx_events(ring);
1364
1365 bcs = tmp >> GEN8_BCS_IRQ_SHIFT;
1366 ring = &dev_priv->ring[BCS];
Ben Widawskyabd58f02013-11-02 21:07:09 -07001367 if (bcs & GT_RENDER_USER_INTERRUPT)
Thomas Daniele981e7b2014-07-24 17:04:39 +01001368 notify_ring(dev, ring);
1369 if (bcs & GT_CONTEXT_SWITCH_INTERRUPT)
1370 intel_execlists_handle_ctx_events(ring);
Ben Widawskyabd58f02013-11-02 21:07:09 -07001371 } else
1372 DRM_ERROR("The master control interrupt lied (GT0)!\n");
1373 }
1374
Zhao Yakui85f9b5f2014-04-17 10:37:38 +08001375 if (master_ctl & (GEN8_GT_VCS1_IRQ | GEN8_GT_VCS2_IRQ)) {
Ben Widawskyabd58f02013-11-02 21:07:09 -07001376 tmp = I915_READ(GEN8_GT_IIR(1));
1377 if (tmp) {
Oscar Mateo38cc46d2014-06-16 16:10:59 +01001378 I915_WRITE(GEN8_GT_IIR(1), tmp);
Ben Widawskyabd58f02013-11-02 21:07:09 -07001379 ret = IRQ_HANDLED;
Thomas Daniele981e7b2014-07-24 17:04:39 +01001380
Ben Widawskyabd58f02013-11-02 21:07:09 -07001381 vcs = tmp >> GEN8_VCS1_IRQ_SHIFT;
Thomas Daniele981e7b2014-07-24 17:04:39 +01001382 ring = &dev_priv->ring[VCS];
Ben Widawskyabd58f02013-11-02 21:07:09 -07001383 if (vcs & GT_RENDER_USER_INTERRUPT)
Thomas Daniele981e7b2014-07-24 17:04:39 +01001384 notify_ring(dev, ring);
Oscar Mateo73d477f2014-07-24 17:04:31 +01001385 if (vcs & GT_CONTEXT_SWITCH_INTERRUPT)
Thomas Daniele981e7b2014-07-24 17:04:39 +01001386 intel_execlists_handle_ctx_events(ring);
1387
Zhao Yakui85f9b5f2014-04-17 10:37:38 +08001388 vcs = tmp >> GEN8_VCS2_IRQ_SHIFT;
Thomas Daniele981e7b2014-07-24 17:04:39 +01001389 ring = &dev_priv->ring[VCS2];
Zhao Yakui85f9b5f2014-04-17 10:37:38 +08001390 if (vcs & GT_RENDER_USER_INTERRUPT)
Thomas Daniele981e7b2014-07-24 17:04:39 +01001391 notify_ring(dev, ring);
Oscar Mateo73d477f2014-07-24 17:04:31 +01001392 if (vcs & GT_CONTEXT_SWITCH_INTERRUPT)
Thomas Daniele981e7b2014-07-24 17:04:39 +01001393 intel_execlists_handle_ctx_events(ring);
Ben Widawskyabd58f02013-11-02 21:07:09 -07001394 } else
1395 DRM_ERROR("The master control interrupt lied (GT1)!\n");
1396 }
1397
Ben Widawsky09610212014-05-15 20:58:08 +03001398 if (master_ctl & GEN8_GT_PM_IRQ) {
1399 tmp = I915_READ(GEN8_GT_IIR(2));
1400 if (tmp & dev_priv->pm_rps_events) {
Ben Widawsky09610212014-05-15 20:58:08 +03001401 I915_WRITE(GEN8_GT_IIR(2),
1402 tmp & dev_priv->pm_rps_events);
Oscar Mateo38cc46d2014-06-16 16:10:59 +01001403 ret = IRQ_HANDLED;
Imre Deakc9a9a262014-11-05 20:48:37 +02001404 gen6_rps_irq_handler(dev_priv, tmp);
Ben Widawsky09610212014-05-15 20:58:08 +03001405 } else
1406 DRM_ERROR("The master control interrupt lied (PM)!\n");
1407 }
1408
Ben Widawskyabd58f02013-11-02 21:07:09 -07001409 if (master_ctl & GEN8_GT_VECS_IRQ) {
1410 tmp = I915_READ(GEN8_GT_IIR(3));
1411 if (tmp) {
Oscar Mateo38cc46d2014-06-16 16:10:59 +01001412 I915_WRITE(GEN8_GT_IIR(3), tmp);
Ben Widawskyabd58f02013-11-02 21:07:09 -07001413 ret = IRQ_HANDLED;
Thomas Daniele981e7b2014-07-24 17:04:39 +01001414
Ben Widawskyabd58f02013-11-02 21:07:09 -07001415 vcs = tmp >> GEN8_VECS_IRQ_SHIFT;
Thomas Daniele981e7b2014-07-24 17:04:39 +01001416 ring = &dev_priv->ring[VECS];
Ben Widawskyabd58f02013-11-02 21:07:09 -07001417 if (vcs & GT_RENDER_USER_INTERRUPT)
Thomas Daniele981e7b2014-07-24 17:04:39 +01001418 notify_ring(dev, ring);
Oscar Mateo73d477f2014-07-24 17:04:31 +01001419 if (vcs & GT_CONTEXT_SWITCH_INTERRUPT)
Thomas Daniele981e7b2014-07-24 17:04:39 +01001420 intel_execlists_handle_ctx_events(ring);
Ben Widawskyabd58f02013-11-02 21:07:09 -07001421 } else
1422 DRM_ERROR("The master control interrupt lied (GT3)!\n");
1423 }
1424
1425 return ret;
1426}
1427
Egbert Eichb543fb02013-04-16 13:36:54 +02001428#define HPD_STORM_DETECT_PERIOD 1000
1429#define HPD_STORM_THRESHOLD 5
1430
Jani Nikula07c338c2014-10-02 11:16:32 +03001431static int pch_port_to_hotplug_shift(enum port port)
Dave Airlie13cf5502014-06-18 11:29:35 +10001432{
1433 switch (port) {
1434 case PORT_A:
1435 case PORT_E:
1436 default:
1437 return -1;
1438 case PORT_B:
1439 return 0;
1440 case PORT_C:
1441 return 8;
1442 case PORT_D:
1443 return 16;
1444 }
1445}
1446
Jani Nikula07c338c2014-10-02 11:16:32 +03001447static int i915_port_to_hotplug_shift(enum port port)
Dave Airlie13cf5502014-06-18 11:29:35 +10001448{
1449 switch (port) {
1450 case PORT_A:
1451 case PORT_E:
1452 default:
1453 return -1;
1454 case PORT_B:
1455 return 17;
1456 case PORT_C:
1457 return 19;
1458 case PORT_D:
1459 return 21;
1460 }
1461}
1462
1463static inline enum port get_port_from_pin(enum hpd_pin pin)
1464{
1465 switch (pin) {
1466 case HPD_PORT_B:
1467 return PORT_B;
1468 case HPD_PORT_C:
1469 return PORT_C;
1470 case HPD_PORT_D:
1471 return PORT_D;
1472 default:
1473 return PORT_A; /* no hpd */
1474 }
1475}
1476
Daniel Vetter10a504d2013-06-27 17:52:12 +02001477static inline void intel_hpd_irq_handler(struct drm_device *dev,
Daniel Vetter22062db2013-06-27 17:52:11 +02001478 u32 hotplug_trigger,
Dave Airlie13cf5502014-06-18 11:29:35 +10001479 u32 dig_hotplug_reg,
Daniel Vetter22062db2013-06-27 17:52:11 +02001480 const u32 *hpd)
Egbert Eichb543fb02013-04-16 13:36:54 +02001481{
Jani Nikula2d1013d2014-03-31 14:27:17 +03001482 struct drm_i915_private *dev_priv = dev->dev_private;
Egbert Eichb543fb02013-04-16 13:36:54 +02001483 int i;
Dave Airlie13cf5502014-06-18 11:29:35 +10001484 enum port port;
Daniel Vetter10a504d2013-06-27 17:52:12 +02001485 bool storm_detected = false;
Dave Airlie13cf5502014-06-18 11:29:35 +10001486 bool queue_dig = false, queue_hp = false;
1487 u32 dig_shift;
1488 u32 dig_port_mask = 0;
Egbert Eichb543fb02013-04-16 13:36:54 +02001489
Daniel Vetter91d131d2013-06-27 17:52:14 +02001490 if (!hotplug_trigger)
1491 return;
1492
Dave Airlie13cf5502014-06-18 11:29:35 +10001493 DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x, dig 0x%08x\n",
1494 hotplug_trigger, dig_hotplug_reg);
Imre Deakcc9bd492014-01-16 19:56:54 +02001495
Daniel Vetterb5ea2d52013-06-27 17:52:15 +02001496 spin_lock(&dev_priv->irq_lock);
Egbert Eichb543fb02013-04-16 13:36:54 +02001497 for (i = 1; i < HPD_NUM_PINS; i++) {
Dave Airlie13cf5502014-06-18 11:29:35 +10001498 if (!(hpd[i] & hotplug_trigger))
1499 continue;
Egbert Eich821450c2013-04-16 13:36:55 +02001500
Dave Airlie13cf5502014-06-18 11:29:35 +10001501 port = get_port_from_pin(i);
1502 if (port && dev_priv->hpd_irq_port[port]) {
1503 bool long_hpd;
1504
Jani Nikula07c338c2014-10-02 11:16:32 +03001505 if (HAS_PCH_SPLIT(dev)) {
1506 dig_shift = pch_port_to_hotplug_shift(port);
Dave Airlie13cf5502014-06-18 11:29:35 +10001507 long_hpd = (dig_hotplug_reg >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT;
Jani Nikula07c338c2014-10-02 11:16:32 +03001508 } else {
1509 dig_shift = i915_port_to_hotplug_shift(port);
1510 long_hpd = (hotplug_trigger >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT;
Dave Airlie13cf5502014-06-18 11:29:35 +10001511 }
1512
Ville Syrjälä26fbb772014-08-11 18:37:37 +03001513 DRM_DEBUG_DRIVER("digital hpd port %c - %s\n",
1514 port_name(port),
1515 long_hpd ? "long" : "short");
Dave Airlie13cf5502014-06-18 11:29:35 +10001516 /* for long HPD pulses we want to have the digital queue happen,
1517 but we still want HPD storm detection to function. */
1518 if (long_hpd) {
1519 dev_priv->long_hpd_port_mask |= (1 << port);
1520 dig_port_mask |= hpd[i];
1521 } else {
1522 /* for short HPD just trigger the digital queue */
1523 dev_priv->short_hpd_port_mask |= (1 << port);
1524 hotplug_trigger &= ~hpd[i];
1525 }
1526 queue_dig = true;
1527 }
1528 }
1529
1530 for (i = 1; i < HPD_NUM_PINS; i++) {
Daniel Vetter3ff04a162014-04-24 12:03:17 +02001531 if (hpd[i] & hotplug_trigger &&
1532 dev_priv->hpd_stats[i].hpd_mark == HPD_DISABLED) {
1533 /*
1534 * On GMCH platforms the interrupt mask bits only
1535 * prevent irq generation, not the setting of the
1536 * hotplug bits itself. So only WARN about unexpected
1537 * interrupts on saner platforms.
1538 */
1539 WARN_ONCE(INTEL_INFO(dev)->gen >= 5 && !IS_VALLEYVIEW(dev),
1540 "Received HPD interrupt (0x%08x) on pin %d (0x%08x) although disabled\n",
1541 hotplug_trigger, i, hpd[i]);
1542
1543 continue;
1544 }
Egbert Eichb8f102e2013-07-26 14:14:24 +02001545
Egbert Eichb543fb02013-04-16 13:36:54 +02001546 if (!(hpd[i] & hotplug_trigger) ||
1547 dev_priv->hpd_stats[i].hpd_mark != HPD_ENABLED)
1548 continue;
1549
Dave Airlie13cf5502014-06-18 11:29:35 +10001550 if (!(dig_port_mask & hpd[i])) {
1551 dev_priv->hpd_event_bits |= (1 << i);
1552 queue_hp = true;
1553 }
1554
Egbert Eichb543fb02013-04-16 13:36:54 +02001555 if (!time_in_range(jiffies, dev_priv->hpd_stats[i].hpd_last_jiffies,
1556 dev_priv->hpd_stats[i].hpd_last_jiffies
1557 + msecs_to_jiffies(HPD_STORM_DETECT_PERIOD))) {
1558 dev_priv->hpd_stats[i].hpd_last_jiffies = jiffies;
1559 dev_priv->hpd_stats[i].hpd_cnt = 0;
Egbert Eichb8f102e2013-07-26 14:14:24 +02001560 DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: 0\n", i);
Egbert Eichb543fb02013-04-16 13:36:54 +02001561 } else if (dev_priv->hpd_stats[i].hpd_cnt > HPD_STORM_THRESHOLD) {
1562 dev_priv->hpd_stats[i].hpd_mark = HPD_MARK_DISABLED;
Egbert Eich142e2392013-04-11 15:57:57 +02001563 dev_priv->hpd_event_bits &= ~(1 << i);
Egbert Eichb543fb02013-04-16 13:36:54 +02001564 DRM_DEBUG_KMS("HPD interrupt storm detected on PIN %d\n", i);
Daniel Vetter10a504d2013-06-27 17:52:12 +02001565 storm_detected = true;
Egbert Eichb543fb02013-04-16 13:36:54 +02001566 } else {
1567 dev_priv->hpd_stats[i].hpd_cnt++;
Egbert Eichb8f102e2013-07-26 14:14:24 +02001568 DRM_DEBUG_KMS("Received HPD interrupt on PIN %d - cnt: %d\n", i,
1569 dev_priv->hpd_stats[i].hpd_cnt);
Egbert Eichb543fb02013-04-16 13:36:54 +02001570 }
1571 }
1572
Daniel Vetter10a504d2013-06-27 17:52:12 +02001573 if (storm_detected)
1574 dev_priv->display.hpd_irq_setup(dev);
Daniel Vetterb5ea2d52013-06-27 17:52:15 +02001575 spin_unlock(&dev_priv->irq_lock);
Daniel Vetter5876fa02013-06-27 17:52:13 +02001576
Daniel Vetter645416f2013-09-02 16:22:25 +02001577 /*
1578 * Our hotplug handler can grab modeset locks (by calling down into the
1579 * fb helpers). Hence it must not be run on our own dev-priv->wq work
1580 * queue for otherwise the flush_work in the pageflip code will
1581 * deadlock.
1582 */
Dave Airlie13cf5502014-06-18 11:29:35 +10001583 if (queue_dig)
Dave Airlie0e32b392014-05-02 14:02:48 +10001584 queue_work(dev_priv->dp_wq, &dev_priv->dig_port_work);
Dave Airlie13cf5502014-06-18 11:29:35 +10001585 if (queue_hp)
1586 schedule_work(&dev_priv->hotplug_work);
Egbert Eichb543fb02013-04-16 13:36:54 +02001587}
1588
Daniel Vetter515ac2b2012-12-01 13:53:44 +01001589static void gmbus_irq_handler(struct drm_device *dev)
1590{
Jani Nikula2d1013d2014-03-31 14:27:17 +03001591 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter28c70f12012-12-01 13:53:45 +01001592
Daniel Vetter28c70f12012-12-01 13:53:45 +01001593 wake_up_all(&dev_priv->gmbus_wait_queue);
Daniel Vetter515ac2b2012-12-01 13:53:44 +01001594}
1595
Daniel Vetterce99c252012-12-01 13:53:47 +01001596static void dp_aux_irq_handler(struct drm_device *dev)
1597{
Jani Nikula2d1013d2014-03-31 14:27:17 +03001598 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter9ee32fea2012-12-01 13:53:48 +01001599
Daniel Vetter9ee32fea2012-12-01 13:53:48 +01001600 wake_up_all(&dev_priv->gmbus_wait_queue);
Daniel Vetterce99c252012-12-01 13:53:47 +01001601}
1602
Shuang He8bf1e9f2013-10-15 18:55:27 +01001603#if defined(CONFIG_DEBUG_FS)
Daniel Vetter277de952013-10-18 16:37:07 +02001604static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1605 uint32_t crc0, uint32_t crc1,
1606 uint32_t crc2, uint32_t crc3,
1607 uint32_t crc4)
Shuang He8bf1e9f2013-10-15 18:55:27 +01001608{
1609 struct drm_i915_private *dev_priv = dev->dev_private;
1610 struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
1611 struct intel_pipe_crc_entry *entry;
Damien Lespiauac2300d2013-10-15 18:55:30 +01001612 int head, tail;
Damien Lespiaub2c88f52013-10-15 18:55:29 +01001613
Damien Lespiaud538bbd2013-10-21 14:29:30 +01001614 spin_lock(&pipe_crc->lock);
1615
Damien Lespiau0c912c72013-10-15 18:55:37 +01001616 if (!pipe_crc->entries) {
Damien Lespiaud538bbd2013-10-21 14:29:30 +01001617 spin_unlock(&pipe_crc->lock);
Damien Lespiau0c912c72013-10-15 18:55:37 +01001618 DRM_ERROR("spurious interrupt\n");
1619 return;
1620 }
1621
Damien Lespiaud538bbd2013-10-21 14:29:30 +01001622 head = pipe_crc->head;
1623 tail = pipe_crc->tail;
Damien Lespiaub2c88f52013-10-15 18:55:29 +01001624
1625 if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
Damien Lespiaud538bbd2013-10-21 14:29:30 +01001626 spin_unlock(&pipe_crc->lock);
Damien Lespiaub2c88f52013-10-15 18:55:29 +01001627 DRM_ERROR("CRC buffer overflowing\n");
1628 return;
1629 }
1630
1631 entry = &pipe_crc->entries[head];
Shuang He8bf1e9f2013-10-15 18:55:27 +01001632
Daniel Vetter8bc5e952013-10-16 22:55:49 +02001633 entry->frame = dev->driver->get_vblank_counter(dev, pipe);
Daniel Vettereba94eb2013-10-16 22:55:46 +02001634 entry->crc[0] = crc0;
1635 entry->crc[1] = crc1;
1636 entry->crc[2] = crc2;
1637 entry->crc[3] = crc3;
1638 entry->crc[4] = crc4;
Damien Lespiaub2c88f52013-10-15 18:55:29 +01001639
1640 head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
Damien Lespiaud538bbd2013-10-21 14:29:30 +01001641 pipe_crc->head = head;
1642
1643 spin_unlock(&pipe_crc->lock);
Damien Lespiau07144422013-10-15 18:55:40 +01001644
1645 wake_up_interruptible(&pipe_crc->wq);
Shuang He8bf1e9f2013-10-15 18:55:27 +01001646}
Daniel Vetter277de952013-10-18 16:37:07 +02001647#else
1648static inline void
1649display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
1650 uint32_t crc0, uint32_t crc1,
1651 uint32_t crc2, uint32_t crc3,
1652 uint32_t crc4) {}
1653#endif
Daniel Vettereba94eb2013-10-16 22:55:46 +02001654
Daniel Vetter277de952013-10-18 16:37:07 +02001655
1656static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
Daniel Vetter5a69b892013-10-16 22:55:52 +02001657{
1658 struct drm_i915_private *dev_priv = dev->dev_private;
1659
Daniel Vetter277de952013-10-18 16:37:07 +02001660 display_pipe_crc_irq_handler(dev, pipe,
1661 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1662 0, 0, 0, 0);
Daniel Vetter5a69b892013-10-16 22:55:52 +02001663}
1664
Daniel Vetter277de952013-10-18 16:37:07 +02001665static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
Daniel Vettereba94eb2013-10-16 22:55:46 +02001666{
1667 struct drm_i915_private *dev_priv = dev->dev_private;
1668
Daniel Vetter277de952013-10-18 16:37:07 +02001669 display_pipe_crc_irq_handler(dev, pipe,
1670 I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
1671 I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
1672 I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
1673 I915_READ(PIPE_CRC_RES_4_IVB(pipe)),
1674 I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
Daniel Vettereba94eb2013-10-16 22:55:46 +02001675}
Daniel Vetter5b3a8562013-10-16 22:55:48 +02001676
Daniel Vetter277de952013-10-18 16:37:07 +02001677static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
Daniel Vetter5b3a8562013-10-16 22:55:48 +02001678{
1679 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter0b5c5ed2013-10-16 22:55:53 +02001680 uint32_t res1, res2;
1681
1682 if (INTEL_INFO(dev)->gen >= 3)
1683 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
1684 else
1685 res1 = 0;
1686
1687 if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
1688 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
1689 else
1690 res2 = 0;
Daniel Vetter5b3a8562013-10-16 22:55:48 +02001691
Daniel Vetter277de952013-10-18 16:37:07 +02001692 display_pipe_crc_irq_handler(dev, pipe,
1693 I915_READ(PIPE_CRC_RES_RED(pipe)),
1694 I915_READ(PIPE_CRC_RES_GREEN(pipe)),
1695 I915_READ(PIPE_CRC_RES_BLUE(pipe)),
1696 res1, res2);
Daniel Vetter5b3a8562013-10-16 22:55:48 +02001697}
Shuang He8bf1e9f2013-10-15 18:55:27 +01001698
Paulo Zanoni1403c0d2013-08-15 11:51:32 -03001699/* The RPS events need forcewake, so we add them to a work queue and mask their
1700 * IMR bits until the work is done. Other interrupts can be processed without
1701 * the work queue. */
1702static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
Ben Widawskybaf02a12013-05-28 19:22:24 -07001703{
Imre Deak4a74de82014-11-19 15:30:01 +02001704 /* TODO: RPS on GEN9+ is not supported yet. */
1705 if (WARN_ONCE(INTEL_INFO(dev_priv)->gen >= 9,
1706 "GEN9+: unexpected RPS IRQ\n"))
Imre Deak132f3f12014-11-10 15:34:33 +02001707 return;
1708
Deepak Sa6706b42014-03-15 20:23:22 +05301709 if (pm_iir & dev_priv->pm_rps_events) {
Daniel Vetter59cdb632013-07-04 23:35:28 +02001710 spin_lock(&dev_priv->irq_lock);
Deepak Sa6706b42014-03-15 20:23:22 +05301711 dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
Daniel Vetter480c8032014-07-16 09:49:40 +02001712 gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
Daniel Vetter59cdb632013-07-04 23:35:28 +02001713 spin_unlock(&dev_priv->irq_lock);
Daniel Vetter2adbee62013-07-04 23:35:27 +02001714
1715 queue_work(dev_priv->wq, &dev_priv->rps.work);
Ben Widawskybaf02a12013-05-28 19:22:24 -07001716 }
Ben Widawskybaf02a12013-05-28 19:22:24 -07001717
Imre Deakc9a9a262014-11-05 20:48:37 +02001718 if (INTEL_INFO(dev_priv)->gen >= 8)
1719 return;
1720
Paulo Zanoni1403c0d2013-08-15 11:51:32 -03001721 if (HAS_VEBOX(dev_priv->dev)) {
1722 if (pm_iir & PM_VEBOX_USER_INTERRUPT)
1723 notify_ring(dev_priv->dev, &dev_priv->ring[VECS]);
Ben Widawsky12638c52013-05-28 19:22:31 -07001724
Paulo Zanoni1403c0d2013-08-15 11:51:32 -03001725 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) {
Mika Kuoppala58174462014-02-25 17:11:26 +02001726 i915_handle_error(dev_priv->dev, false,
1727 "VEBOX CS error interrupt 0x%08x",
1728 pm_iir);
Paulo Zanoni1403c0d2013-08-15 11:51:32 -03001729 }
Ben Widawsky12638c52013-05-28 19:22:31 -07001730 }
Ben Widawskybaf02a12013-05-28 19:22:24 -07001731}
1732
Ville Syrjälä8d7849d2014-04-29 13:35:46 +03001733static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
1734{
Ville Syrjälä8d7849d2014-04-29 13:35:46 +03001735 if (!drm_handle_vblank(dev, pipe))
1736 return false;
1737
Ville Syrjälä8d7849d2014-04-29 13:35:46 +03001738 return true;
1739}
1740
Imre Deakc1874ed2014-02-04 21:35:46 +02001741static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
1742{
1743 struct drm_i915_private *dev_priv = dev->dev_private;
Imre Deak91d181d2014-02-10 18:42:49 +02001744 u32 pipe_stats[I915_MAX_PIPES] = { };
Imre Deakc1874ed2014-02-04 21:35:46 +02001745 int pipe;
1746
Imre Deak58ead0d2014-02-04 21:35:47 +02001747 spin_lock(&dev_priv->irq_lock);
Damien Lespiau055e3932014-08-18 13:49:10 +01001748 for_each_pipe(dev_priv, pipe) {
Imre Deak91d181d2014-02-10 18:42:49 +02001749 int reg;
Daniel Vetterbbb5eeb2014-02-12 17:55:36 +01001750 u32 mask, iir_bit = 0;
Imre Deak91d181d2014-02-10 18:42:49 +02001751
Daniel Vetterbbb5eeb2014-02-12 17:55:36 +01001752 /*
1753 * PIPESTAT bits get signalled even when the interrupt is
1754 * disabled with the mask bits, and some of the status bits do
1755 * not generate interrupts at all (like the underrun bit). Hence
1756 * we need to be careful that we only handle what we want to
1757 * handle.
1758 */
Daniel Vetter0f239f42014-09-30 10:56:49 +02001759
1760 /* fifo underruns are filterered in the underrun handler. */
1761 mask = PIPE_FIFO_UNDERRUN_STATUS;
Daniel Vetterbbb5eeb2014-02-12 17:55:36 +01001762
1763 switch (pipe) {
1764 case PIPE_A:
1765 iir_bit = I915_DISPLAY_PIPE_A_EVENT_INTERRUPT;
1766 break;
1767 case PIPE_B:
1768 iir_bit = I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
1769 break;
Ville Syrjälä3278f672014-04-09 13:28:49 +03001770 case PIPE_C:
1771 iir_bit = I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
1772 break;
Daniel Vetterbbb5eeb2014-02-12 17:55:36 +01001773 }
1774 if (iir & iir_bit)
1775 mask |= dev_priv->pipestat_irq_mask[pipe];
1776
1777 if (!mask)
Imre Deak91d181d2014-02-10 18:42:49 +02001778 continue;
1779
1780 reg = PIPESTAT(pipe);
Daniel Vetterbbb5eeb2014-02-12 17:55:36 +01001781 mask |= PIPESTAT_INT_ENABLE_MASK;
1782 pipe_stats[pipe] = I915_READ(reg) & mask;
Imre Deakc1874ed2014-02-04 21:35:46 +02001783
1784 /*
1785 * Clear the PIPE*STAT regs before the IIR
1786 */
Imre Deak91d181d2014-02-10 18:42:49 +02001787 if (pipe_stats[pipe] & (PIPE_FIFO_UNDERRUN_STATUS |
1788 PIPESTAT_INT_STATUS_MASK))
Imre Deakc1874ed2014-02-04 21:35:46 +02001789 I915_WRITE(reg, pipe_stats[pipe]);
1790 }
Imre Deak58ead0d2014-02-04 21:35:47 +02001791 spin_unlock(&dev_priv->irq_lock);
Imre Deakc1874ed2014-02-04 21:35:46 +02001792
Damien Lespiau055e3932014-08-18 13:49:10 +01001793 for_each_pipe(dev_priv, pipe) {
Chris Wilsond6bbafa2014-09-05 07:13:24 +01001794 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
1795 intel_pipe_handle_vblank(dev, pipe))
1796 intel_check_page_flip(dev, pipe);
Imre Deakc1874ed2014-02-04 21:35:46 +02001797
Imre Deak579a9b02014-02-04 21:35:48 +02001798 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
Imre Deakc1874ed2014-02-04 21:35:46 +02001799 intel_prepare_page_flip(dev, pipe);
1800 intel_finish_page_flip(dev, pipe);
1801 }
1802
1803 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
1804 i9xx_pipe_crc_irq_handler(dev, pipe);
1805
Daniel Vetter1f7247c2014-09-30 10:56:48 +02001806 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
1807 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
Imre Deakc1874ed2014-02-04 21:35:46 +02001808 }
1809
1810 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
1811 gmbus_irq_handler(dev);
1812}
1813
Ville Syrjälä16c6c562014-04-01 10:54:36 +03001814static void i9xx_hpd_irq_handler(struct drm_device *dev)
1815{
1816 struct drm_i915_private *dev_priv = dev->dev_private;
1817 u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
1818
Oscar Mateo3ff60f82014-06-16 16:10:58 +01001819 if (hotplug_status) {
1820 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
1821 /*
1822 * Make sure hotplug status is cleared before we clear IIR, or else we
1823 * may miss hotplug events.
1824 */
1825 POSTING_READ(PORT_HOTPLUG_STAT);
Ville Syrjälä16c6c562014-04-01 10:54:36 +03001826
Oscar Mateo3ff60f82014-06-16 16:10:58 +01001827 if (IS_G4X(dev)) {
1828 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
Ville Syrjälä16c6c562014-04-01 10:54:36 +03001829
Dave Airlie13cf5502014-06-18 11:29:35 +10001830 intel_hpd_irq_handler(dev, hotplug_trigger, 0, hpd_status_g4x);
Oscar Mateo3ff60f82014-06-16 16:10:58 +01001831 } else {
1832 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
1833
Dave Airlie13cf5502014-06-18 11:29:35 +10001834 intel_hpd_irq_handler(dev, hotplug_trigger, 0, hpd_status_i915);
Oscar Mateo3ff60f82014-06-16 16:10:58 +01001835 }
1836
1837 if ((IS_G4X(dev) || IS_VALLEYVIEW(dev)) &&
1838 hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
1839 dp_aux_irq_handler(dev);
Ville Syrjälä16c6c562014-04-01 10:54:36 +03001840 }
Ville Syrjälä16c6c562014-04-01 10:54:36 +03001841}
1842
Daniel Vetterff1f5252012-10-02 15:10:55 +02001843static irqreturn_t valleyview_irq_handler(int irq, void *arg)
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07001844{
Daniel Vetter45a83f82014-05-12 19:17:55 +02001845 struct drm_device *dev = arg;
Jani Nikula2d1013d2014-03-31 14:27:17 +03001846 struct drm_i915_private *dev_priv = dev->dev_private;
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07001847 u32 iir, gt_iir, pm_iir;
1848 irqreturn_t ret = IRQ_NONE;
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07001849
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07001850 while (true) {
Oscar Mateo3ff60f82014-06-16 16:10:58 +01001851 /* Find, clear, then process each source of interrupt */
1852
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07001853 gt_iir = I915_READ(GTIIR);
Oscar Mateo3ff60f82014-06-16 16:10:58 +01001854 if (gt_iir)
1855 I915_WRITE(GTIIR, gt_iir);
1856
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07001857 pm_iir = I915_READ(GEN6_PMIIR);
Oscar Mateo3ff60f82014-06-16 16:10:58 +01001858 if (pm_iir)
1859 I915_WRITE(GEN6_PMIIR, pm_iir);
1860
1861 iir = I915_READ(VLV_IIR);
1862 if (iir) {
1863 /* Consume port before clearing IIR or we'll miss events */
1864 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1865 i9xx_hpd_irq_handler(dev);
1866 I915_WRITE(VLV_IIR, iir);
1867 }
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07001868
1869 if (gt_iir == 0 && pm_iir == 0 && iir == 0)
1870 goto out;
1871
1872 ret = IRQ_HANDLED;
1873
Oscar Mateo3ff60f82014-06-16 16:10:58 +01001874 if (gt_iir)
1875 snb_gt_irq_handler(dev, dev_priv, gt_iir);
Paulo Zanoni60611c12013-08-15 11:50:01 -03001876 if (pm_iir)
Daniel Vetterd0ecd7e2013-07-04 23:35:25 +02001877 gen6_rps_irq_handler(dev_priv, pm_iir);
Oscar Mateo3ff60f82014-06-16 16:10:58 +01001878 /* Call regardless, as some status bits might not be
1879 * signalled in iir */
1880 valleyview_pipestat_irq_handler(dev, iir);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07001881 }
1882
1883out:
1884 return ret;
1885}
1886
Ville Syrjälä43f328d2014-04-09 20:40:52 +03001887static irqreturn_t cherryview_irq_handler(int irq, void *arg)
1888{
Daniel Vetter45a83f82014-05-12 19:17:55 +02001889 struct drm_device *dev = arg;
Ville Syrjälä43f328d2014-04-09 20:40:52 +03001890 struct drm_i915_private *dev_priv = dev->dev_private;
1891 u32 master_ctl, iir;
1892 irqreturn_t ret = IRQ_NONE;
Ville Syrjälä43f328d2014-04-09 20:40:52 +03001893
Ville Syrjälä8e5fd592014-04-09 13:28:50 +03001894 for (;;) {
1895 master_ctl = I915_READ(GEN8_MASTER_IRQ) & ~GEN8_MASTER_IRQ_CONTROL;
1896 iir = I915_READ(VLV_IIR);
Ville Syrjälä3278f672014-04-09 13:28:49 +03001897
Ville Syrjälä8e5fd592014-04-09 13:28:50 +03001898 if (master_ctl == 0 && iir == 0)
1899 break;
Ville Syrjälä43f328d2014-04-09 20:40:52 +03001900
Oscar Mateo27b6c122014-06-16 16:11:00 +01001901 ret = IRQ_HANDLED;
1902
Ville Syrjälä8e5fd592014-04-09 13:28:50 +03001903 I915_WRITE(GEN8_MASTER_IRQ, 0);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03001904
Oscar Mateo27b6c122014-06-16 16:11:00 +01001905 /* Find, clear, then process each source of interrupt */
1906
1907 if (iir) {
1908 /* Consume port before clearing IIR or we'll miss events */
1909 if (iir & I915_DISPLAY_PORT_INTERRUPT)
1910 i9xx_hpd_irq_handler(dev);
1911 I915_WRITE(VLV_IIR, iir);
1912 }
1913
Ville Syrjälä8e5fd592014-04-09 13:28:50 +03001914 gen8_gt_irq_handler(dev, dev_priv, master_ctl);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03001915
Oscar Mateo27b6c122014-06-16 16:11:00 +01001916 /* Call regardless, as some status bits might not be
1917 * signalled in iir */
Ville Syrjälä8e5fd592014-04-09 13:28:50 +03001918 valleyview_pipestat_irq_handler(dev, iir);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03001919
Ville Syrjälä8e5fd592014-04-09 13:28:50 +03001920 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
1921 POSTING_READ(GEN8_MASTER_IRQ);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03001922 }
1923
Ville Syrjälä43f328d2014-04-09 20:40:52 +03001924 return ret;
1925}
1926
Adam Jackson23e81d62012-06-06 15:45:44 -04001927static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
Jesse Barnes776ad802011-01-04 15:09:39 -08001928{
Jani Nikula2d1013d2014-03-31 14:27:17 +03001929 struct drm_i915_private *dev_priv = dev->dev_private;
Jesse Barnes9db4a9c2011-02-07 12:26:52 -08001930 int pipe;
Egbert Eichb543fb02013-04-16 13:36:54 +02001931 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
Dave Airlie13cf5502014-06-18 11:29:35 +10001932 u32 dig_hotplug_reg;
Jesse Barnes776ad802011-01-04 15:09:39 -08001933
Dave Airlie13cf5502014-06-18 11:29:35 +10001934 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
1935 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
1936
1937 intel_hpd_irq_handler(dev, hotplug_trigger, dig_hotplug_reg, hpd_ibx);
Daniel Vetter91d131d2013-06-27 17:52:14 +02001938
Ville Syrjäläcfc33bf2013-04-17 17:48:48 +03001939 if (pch_iir & SDE_AUDIO_POWER_MASK) {
1940 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
1941 SDE_AUDIO_POWER_SHIFT);
Jesse Barnes776ad802011-01-04 15:09:39 -08001942 DRM_DEBUG_DRIVER("PCH audio power change on port %d\n",
Ville Syrjäläcfc33bf2013-04-17 17:48:48 +03001943 port_name(port));
1944 }
Jesse Barnes776ad802011-01-04 15:09:39 -08001945
Daniel Vetterce99c252012-12-01 13:53:47 +01001946 if (pch_iir & SDE_AUX_MASK)
1947 dp_aux_irq_handler(dev);
1948
Jesse Barnes776ad802011-01-04 15:09:39 -08001949 if (pch_iir & SDE_GMBUS)
Daniel Vetter515ac2b2012-12-01 13:53:44 +01001950 gmbus_irq_handler(dev);
Jesse Barnes776ad802011-01-04 15:09:39 -08001951
1952 if (pch_iir & SDE_AUDIO_HDCP_MASK)
1953 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
1954
1955 if (pch_iir & SDE_AUDIO_TRANS_MASK)
1956 DRM_DEBUG_DRIVER("PCH transcoder audio interrupt\n");
1957
1958 if (pch_iir & SDE_POISON)
1959 DRM_ERROR("PCH poison interrupt\n");
1960
Jesse Barnes9db4a9c2011-02-07 12:26:52 -08001961 if (pch_iir & SDE_FDI_MASK)
Damien Lespiau055e3932014-08-18 13:49:10 +01001962 for_each_pipe(dev_priv, pipe)
Jesse Barnes9db4a9c2011-02-07 12:26:52 -08001963 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
1964 pipe_name(pipe),
1965 I915_READ(FDI_RX_IIR(pipe)));
Jesse Barnes776ad802011-01-04 15:09:39 -08001966
1967 if (pch_iir & (SDE_TRANSB_CRC_DONE | SDE_TRANSA_CRC_DONE))
1968 DRM_DEBUG_DRIVER("PCH transcoder CRC done interrupt\n");
1969
1970 if (pch_iir & (SDE_TRANSB_CRC_ERR | SDE_TRANSA_CRC_ERR))
1971 DRM_DEBUG_DRIVER("PCH transcoder CRC error interrupt\n");
1972
Jesse Barnes776ad802011-01-04 15:09:39 -08001973 if (pch_iir & SDE_TRANSA_FIFO_UNDER)
Daniel Vetter1f7247c2014-09-30 10:56:48 +02001974 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
Paulo Zanoni86642812013-04-12 17:57:57 -03001975
1976 if (pch_iir & SDE_TRANSB_FIFO_UNDER)
Daniel Vetter1f7247c2014-09-30 10:56:48 +02001977 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
Paulo Zanoni86642812013-04-12 17:57:57 -03001978}
1979
1980static void ivb_err_int_handler(struct drm_device *dev)
1981{
1982 struct drm_i915_private *dev_priv = dev->dev_private;
1983 u32 err_int = I915_READ(GEN7_ERR_INT);
Daniel Vetter5a69b892013-10-16 22:55:52 +02001984 enum pipe pipe;
Paulo Zanoni86642812013-04-12 17:57:57 -03001985
Paulo Zanonide032bf2013-04-12 17:57:58 -03001986 if (err_int & ERR_INT_POISON)
1987 DRM_ERROR("Poison interrupt\n");
1988
Damien Lespiau055e3932014-08-18 13:49:10 +01001989 for_each_pipe(dev_priv, pipe) {
Daniel Vetter1f7247c2014-09-30 10:56:48 +02001990 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe))
1991 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
Paulo Zanoni86642812013-04-12 17:57:57 -03001992
Daniel Vetter5a69b892013-10-16 22:55:52 +02001993 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
1994 if (IS_IVYBRIDGE(dev))
Daniel Vetter277de952013-10-18 16:37:07 +02001995 ivb_pipe_crc_irq_handler(dev, pipe);
Daniel Vetter5a69b892013-10-16 22:55:52 +02001996 else
Daniel Vetter277de952013-10-18 16:37:07 +02001997 hsw_pipe_crc_irq_handler(dev, pipe);
Daniel Vetter5a69b892013-10-16 22:55:52 +02001998 }
1999 }
Shuang He8bf1e9f2013-10-15 18:55:27 +01002000
Paulo Zanoni86642812013-04-12 17:57:57 -03002001 I915_WRITE(GEN7_ERR_INT, err_int);
2002}
2003
2004static void cpt_serr_int_handler(struct drm_device *dev)
2005{
2006 struct drm_i915_private *dev_priv = dev->dev_private;
2007 u32 serr_int = I915_READ(SERR_INT);
2008
Paulo Zanonide032bf2013-04-12 17:57:58 -03002009 if (serr_int & SERR_INT_POISON)
2010 DRM_ERROR("PCH poison interrupt\n");
2011
Paulo Zanoni86642812013-04-12 17:57:57 -03002012 if (serr_int & SERR_INT_TRANS_A_FIFO_UNDERRUN)
Daniel Vetter1f7247c2014-09-30 10:56:48 +02002013 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_A);
Paulo Zanoni86642812013-04-12 17:57:57 -03002014
2015 if (serr_int & SERR_INT_TRANS_B_FIFO_UNDERRUN)
Daniel Vetter1f7247c2014-09-30 10:56:48 +02002016 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
Paulo Zanoni86642812013-04-12 17:57:57 -03002017
2018 if (serr_int & SERR_INT_TRANS_C_FIFO_UNDERRUN)
Daniel Vetter1f7247c2014-09-30 10:56:48 +02002019 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_C);
Paulo Zanoni86642812013-04-12 17:57:57 -03002020
2021 I915_WRITE(SERR_INT, serr_int);
Jesse Barnes776ad802011-01-04 15:09:39 -08002022}
2023
Adam Jackson23e81d62012-06-06 15:45:44 -04002024static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
2025{
Jani Nikula2d1013d2014-03-31 14:27:17 +03002026 struct drm_i915_private *dev_priv = dev->dev_private;
Adam Jackson23e81d62012-06-06 15:45:44 -04002027 int pipe;
Egbert Eichb543fb02013-04-16 13:36:54 +02002028 u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
Dave Airlie13cf5502014-06-18 11:29:35 +10002029 u32 dig_hotplug_reg;
Adam Jackson23e81d62012-06-06 15:45:44 -04002030
Dave Airlie13cf5502014-06-18 11:29:35 +10002031 dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
2032 I915_WRITE(PCH_PORT_HOTPLUG, dig_hotplug_reg);
2033
2034 intel_hpd_irq_handler(dev, hotplug_trigger, dig_hotplug_reg, hpd_cpt);
Daniel Vetter91d131d2013-06-27 17:52:14 +02002035
Ville Syrjäläcfc33bf2013-04-17 17:48:48 +03002036 if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
2037 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
2038 SDE_AUDIO_POWER_SHIFT_CPT);
2039 DRM_DEBUG_DRIVER("PCH audio power change on port %c\n",
2040 port_name(port));
2041 }
Adam Jackson23e81d62012-06-06 15:45:44 -04002042
2043 if (pch_iir & SDE_AUX_MASK_CPT)
Daniel Vetterce99c252012-12-01 13:53:47 +01002044 dp_aux_irq_handler(dev);
Adam Jackson23e81d62012-06-06 15:45:44 -04002045
2046 if (pch_iir & SDE_GMBUS_CPT)
Daniel Vetter515ac2b2012-12-01 13:53:44 +01002047 gmbus_irq_handler(dev);
Adam Jackson23e81d62012-06-06 15:45:44 -04002048
2049 if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
2050 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
2051
2052 if (pch_iir & SDE_AUDIO_CP_CHG_CPT)
2053 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
2054
2055 if (pch_iir & SDE_FDI_MASK_CPT)
Damien Lespiau055e3932014-08-18 13:49:10 +01002056 for_each_pipe(dev_priv, pipe)
Adam Jackson23e81d62012-06-06 15:45:44 -04002057 DRM_DEBUG_DRIVER(" pipe %c FDI IIR: 0x%08x\n",
2058 pipe_name(pipe),
2059 I915_READ(FDI_RX_IIR(pipe)));
Paulo Zanoni86642812013-04-12 17:57:57 -03002060
2061 if (pch_iir & SDE_ERROR_CPT)
2062 cpt_serr_int_handler(dev);
Adam Jackson23e81d62012-06-06 15:45:44 -04002063}
2064
Paulo Zanonic008bc62013-07-12 16:35:10 -03002065static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
2066{
2067 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter40da17c2013-10-21 18:04:36 +02002068 enum pipe pipe;
Paulo Zanonic008bc62013-07-12 16:35:10 -03002069
2070 if (de_iir & DE_AUX_CHANNEL_A)
2071 dp_aux_irq_handler(dev);
2072
2073 if (de_iir & DE_GSE)
2074 intel_opregion_asle_intr(dev);
2075
Paulo Zanonic008bc62013-07-12 16:35:10 -03002076 if (de_iir & DE_POISON)
2077 DRM_ERROR("Poison interrupt\n");
2078
Damien Lespiau055e3932014-08-18 13:49:10 +01002079 for_each_pipe(dev_priv, pipe) {
Chris Wilsond6bbafa2014-09-05 07:13:24 +01002080 if (de_iir & DE_PIPE_VBLANK(pipe) &&
2081 intel_pipe_handle_vblank(dev, pipe))
2082 intel_check_page_flip(dev, pipe);
Paulo Zanonic008bc62013-07-12 16:35:10 -03002083
Daniel Vetter40da17c2013-10-21 18:04:36 +02002084 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
Daniel Vetter1f7247c2014-09-30 10:56:48 +02002085 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
Paulo Zanonic008bc62013-07-12 16:35:10 -03002086
Daniel Vetter40da17c2013-10-21 18:04:36 +02002087 if (de_iir & DE_PIPE_CRC_DONE(pipe))
2088 i9xx_pipe_crc_irq_handler(dev, pipe);
Daniel Vetter5b3a8562013-10-16 22:55:48 +02002089
Daniel Vetter40da17c2013-10-21 18:04:36 +02002090 /* plane/pipes map 1:1 on ilk+ */
2091 if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
2092 intel_prepare_page_flip(dev, pipe);
2093 intel_finish_page_flip_plane(dev, pipe);
2094 }
Paulo Zanonic008bc62013-07-12 16:35:10 -03002095 }
2096
2097 /* check event from PCH */
2098 if (de_iir & DE_PCH_EVENT) {
2099 u32 pch_iir = I915_READ(SDEIIR);
2100
2101 if (HAS_PCH_CPT(dev))
2102 cpt_irq_handler(dev, pch_iir);
2103 else
2104 ibx_irq_handler(dev, pch_iir);
2105
2106 /* should clear PCH hotplug event before clear CPU irq */
2107 I915_WRITE(SDEIIR, pch_iir);
2108 }
2109
2110 if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
2111 ironlake_rps_change_irq_handler(dev);
2112}
2113
Paulo Zanoni9719fb92013-07-12 16:35:11 -03002114static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
2115{
2116 struct drm_i915_private *dev_priv = dev->dev_private;
Damien Lespiau07d27e22014-03-03 17:31:46 +00002117 enum pipe pipe;
Paulo Zanoni9719fb92013-07-12 16:35:11 -03002118
2119 if (de_iir & DE_ERR_INT_IVB)
2120 ivb_err_int_handler(dev);
2121
2122 if (de_iir & DE_AUX_CHANNEL_A_IVB)
2123 dp_aux_irq_handler(dev);
2124
2125 if (de_iir & DE_GSE_IVB)
2126 intel_opregion_asle_intr(dev);
2127
Damien Lespiau055e3932014-08-18 13:49:10 +01002128 for_each_pipe(dev_priv, pipe) {
Chris Wilsond6bbafa2014-09-05 07:13:24 +01002129 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
2130 intel_pipe_handle_vblank(dev, pipe))
2131 intel_check_page_flip(dev, pipe);
Daniel Vetter40da17c2013-10-21 18:04:36 +02002132
2133 /* plane/pipes map 1:1 on ilk+ */
Damien Lespiau07d27e22014-03-03 17:31:46 +00002134 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
2135 intel_prepare_page_flip(dev, pipe);
2136 intel_finish_page_flip_plane(dev, pipe);
Paulo Zanoni9719fb92013-07-12 16:35:11 -03002137 }
2138 }
2139
2140 /* check event from PCH */
2141 if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
2142 u32 pch_iir = I915_READ(SDEIIR);
2143
2144 cpt_irq_handler(dev, pch_iir);
2145
2146 /* clear PCH hotplug event before clear CPU irq */
2147 I915_WRITE(SDEIIR, pch_iir);
2148 }
2149}
2150
Oscar Mateo72c90f62014-06-16 16:10:57 +01002151/*
2152 * To handle irqs with the minimum potential races with fresh interrupts, we:
2153 * 1 - Disable Master Interrupt Control.
2154 * 2 - Find the source(s) of the interrupt.
2155 * 3 - Clear the Interrupt Identity bits (IIR).
2156 * 4 - Process the interrupt(s) that had bits set in the IIRs.
2157 * 5 - Re-enable Master Interrupt Control.
2158 */
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002159static irqreturn_t ironlake_irq_handler(int irq, void *arg)
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002160{
Daniel Vetter45a83f82014-05-12 19:17:55 +02002161 struct drm_device *dev = arg;
Jani Nikula2d1013d2014-03-31 14:27:17 +03002162 struct drm_i915_private *dev_priv = dev->dev_private;
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002163 u32 de_iir, gt_iir, de_ier, sde_ier = 0;
Chris Wilson0e434062012-05-09 21:45:44 +01002164 irqreturn_t ret = IRQ_NONE;
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002165
Paulo Zanoni86642812013-04-12 17:57:57 -03002166 /* We get interrupts on unclaimed registers, so check for this before we
2167 * do any I915_{READ,WRITE}. */
Chris Wilson907b28c2013-07-19 20:36:52 +01002168 intel_uncore_check_errors(dev);
Paulo Zanoni86642812013-04-12 17:57:57 -03002169
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002170 /* disable master interrupt before clearing iir */
2171 de_ier = I915_READ(DEIER);
2172 I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
Paulo Zanoni23a78512013-07-12 16:35:14 -03002173 POSTING_READ(DEIER);
Chris Wilson0e434062012-05-09 21:45:44 +01002174
Paulo Zanoni44498ae2013-02-22 17:05:28 -03002175 /* Disable south interrupts. We'll only write to SDEIIR once, so further
2176 * interrupts will will be stored on its back queue, and then we'll be
2177 * able to process them after we restore SDEIER (as soon as we restore
2178 * it, we'll get an interrupt if SDEIIR still has something to process
2179 * due to its back queue). */
Ben Widawskyab5c6082013-04-05 13:12:41 -07002180 if (!HAS_PCH_NOP(dev)) {
2181 sde_ier = I915_READ(SDEIER);
2182 I915_WRITE(SDEIER, 0);
2183 POSTING_READ(SDEIER);
2184 }
Paulo Zanoni44498ae2013-02-22 17:05:28 -03002185
Oscar Mateo72c90f62014-06-16 16:10:57 +01002186 /* Find, clear, then process each source of interrupt */
2187
Chris Wilson0e434062012-05-09 21:45:44 +01002188 gt_iir = I915_READ(GTIIR);
2189 if (gt_iir) {
Oscar Mateo72c90f62014-06-16 16:10:57 +01002190 I915_WRITE(GTIIR, gt_iir);
2191 ret = IRQ_HANDLED;
Paulo Zanonid8fc8a42013-07-19 18:57:55 -03002192 if (INTEL_INFO(dev)->gen >= 6)
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002193 snb_gt_irq_handler(dev, dev_priv, gt_iir);
Paulo Zanonid8fc8a42013-07-19 18:57:55 -03002194 else
2195 ilk_gt_irq_handler(dev, dev_priv, gt_iir);
Chris Wilson0e434062012-05-09 21:45:44 +01002196 }
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002197
2198 de_iir = I915_READ(DEIIR);
Chris Wilson0e434062012-05-09 21:45:44 +01002199 if (de_iir) {
Oscar Mateo72c90f62014-06-16 16:10:57 +01002200 I915_WRITE(DEIIR, de_iir);
2201 ret = IRQ_HANDLED;
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002202 if (INTEL_INFO(dev)->gen >= 7)
2203 ivb_display_irq_handler(dev, de_iir);
2204 else
2205 ilk_display_irq_handler(dev, de_iir);
Chris Wilson0e434062012-05-09 21:45:44 +01002206 }
2207
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002208 if (INTEL_INFO(dev)->gen >= 6) {
2209 u32 pm_iir = I915_READ(GEN6_PMIIR);
2210 if (pm_iir) {
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002211 I915_WRITE(GEN6_PMIIR, pm_iir);
2212 ret = IRQ_HANDLED;
Oscar Mateo72c90f62014-06-16 16:10:57 +01002213 gen6_rps_irq_handler(dev_priv, pm_iir);
Paulo Zanonif1af8fc2013-07-12 19:56:30 -03002214 }
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002215 }
2216
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002217 I915_WRITE(DEIER, de_ier);
2218 POSTING_READ(DEIER);
Ben Widawskyab5c6082013-04-05 13:12:41 -07002219 if (!HAS_PCH_NOP(dev)) {
2220 I915_WRITE(SDEIER, sde_ier);
2221 POSTING_READ(SDEIER);
2222 }
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002223
2224 return ret;
2225}
2226
Ben Widawskyabd58f02013-11-02 21:07:09 -07002227static irqreturn_t gen8_irq_handler(int irq, void *arg)
2228{
2229 struct drm_device *dev = arg;
2230 struct drm_i915_private *dev_priv = dev->dev_private;
2231 u32 master_ctl;
2232 irqreturn_t ret = IRQ_NONE;
2233 uint32_t tmp = 0;
Daniel Vetterc42664c2013-11-07 11:05:40 +01002234 enum pipe pipe;
Jesse Barnes88e04702014-11-13 17:51:48 +00002235 u32 aux_mask = GEN8_AUX_CHANNEL_A;
2236
2237 if (IS_GEN9(dev))
2238 aux_mask |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
2239 GEN9_AUX_CHANNEL_D;
Ben Widawskyabd58f02013-11-02 21:07:09 -07002240
Ben Widawskyabd58f02013-11-02 21:07:09 -07002241 master_ctl = I915_READ(GEN8_MASTER_IRQ);
2242 master_ctl &= ~GEN8_MASTER_IRQ_CONTROL;
2243 if (!master_ctl)
2244 return IRQ_NONE;
2245
2246 I915_WRITE(GEN8_MASTER_IRQ, 0);
2247 POSTING_READ(GEN8_MASTER_IRQ);
2248
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002249 /* Find, clear, then process each source of interrupt */
2250
Ben Widawskyabd58f02013-11-02 21:07:09 -07002251 ret = gen8_gt_irq_handler(dev, dev_priv, master_ctl);
2252
2253 if (master_ctl & GEN8_DE_MISC_IRQ) {
2254 tmp = I915_READ(GEN8_DE_MISC_IIR);
Ben Widawskyabd58f02013-11-02 21:07:09 -07002255 if (tmp) {
2256 I915_WRITE(GEN8_DE_MISC_IIR, tmp);
2257 ret = IRQ_HANDLED;
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002258 if (tmp & GEN8_DE_MISC_GSE)
2259 intel_opregion_asle_intr(dev);
2260 else
2261 DRM_ERROR("Unexpected DE Misc interrupt\n");
Ben Widawskyabd58f02013-11-02 21:07:09 -07002262 }
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002263 else
2264 DRM_ERROR("The master control interrupt lied (DE MISC)!\n");
Ben Widawskyabd58f02013-11-02 21:07:09 -07002265 }
2266
Daniel Vetter6d766f02013-11-07 14:49:55 +01002267 if (master_ctl & GEN8_DE_PORT_IRQ) {
2268 tmp = I915_READ(GEN8_DE_PORT_IIR);
Daniel Vetter6d766f02013-11-07 14:49:55 +01002269 if (tmp) {
2270 I915_WRITE(GEN8_DE_PORT_IIR, tmp);
2271 ret = IRQ_HANDLED;
Jesse Barnes88e04702014-11-13 17:51:48 +00002272
2273 if (tmp & aux_mask)
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002274 dp_aux_irq_handler(dev);
2275 else
2276 DRM_ERROR("Unexpected DE Port interrupt\n");
Daniel Vetter6d766f02013-11-07 14:49:55 +01002277 }
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002278 else
2279 DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
Daniel Vetter6d766f02013-11-07 14:49:55 +01002280 }
2281
Damien Lespiau055e3932014-08-18 13:49:10 +01002282 for_each_pipe(dev_priv, pipe) {
Damien Lespiau770de832014-03-20 20:45:01 +00002283 uint32_t pipe_iir, flip_done = 0, fault_errors = 0;
Ben Widawskyabd58f02013-11-02 21:07:09 -07002284
Daniel Vetterc42664c2013-11-07 11:05:40 +01002285 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
2286 continue;
Ben Widawskyabd58f02013-11-02 21:07:09 -07002287
Daniel Vetterc42664c2013-11-07 11:05:40 +01002288 pipe_iir = I915_READ(GEN8_DE_PIPE_IIR(pipe));
Daniel Vetterc42664c2013-11-07 11:05:40 +01002289 if (pipe_iir) {
2290 ret = IRQ_HANDLED;
2291 I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
Damien Lespiau770de832014-03-20 20:45:01 +00002292
Chris Wilsond6bbafa2014-09-05 07:13:24 +01002293 if (pipe_iir & GEN8_PIPE_VBLANK &&
2294 intel_pipe_handle_vblank(dev, pipe))
2295 intel_check_page_flip(dev, pipe);
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002296
Damien Lespiau770de832014-03-20 20:45:01 +00002297 if (IS_GEN9(dev))
2298 flip_done = pipe_iir & GEN9_PIPE_PLANE1_FLIP_DONE;
2299 else
2300 flip_done = pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE;
2301
2302 if (flip_done) {
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002303 intel_prepare_page_flip(dev, pipe);
2304 intel_finish_page_flip_plane(dev, pipe);
2305 }
2306
2307 if (pipe_iir & GEN8_PIPE_CDCLK_CRC_DONE)
2308 hsw_pipe_crc_irq_handler(dev, pipe);
2309
Daniel Vetter1f7247c2014-09-30 10:56:48 +02002310 if (pipe_iir & GEN8_PIPE_FIFO_UNDERRUN)
2311 intel_cpu_fifo_underrun_irq_handler(dev_priv,
2312 pipe);
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002313
Damien Lespiau770de832014-03-20 20:45:01 +00002314
2315 if (IS_GEN9(dev))
2316 fault_errors = pipe_iir & GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
2317 else
2318 fault_errors = pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
2319
2320 if (fault_errors)
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002321 DRM_ERROR("Fault errors on pipe %c\n: 0x%08x",
2322 pipe_name(pipe),
2323 pipe_iir & GEN8_DE_PIPE_IRQ_FAULT_ERRORS);
Daniel Vetterc42664c2013-11-07 11:05:40 +01002324 } else
Ben Widawskyabd58f02013-11-02 21:07:09 -07002325 DRM_ERROR("The master control interrupt lied (DE PIPE)!\n");
2326 }
2327
Daniel Vetter92d03a82013-11-07 11:05:43 +01002328 if (!HAS_PCH_NOP(dev) && master_ctl & GEN8_DE_PCH_IRQ) {
2329 /*
2330 * FIXME(BDW): Assume for now that the new interrupt handling
2331 * scheme also closed the SDE interrupt handling race we've seen
2332 * on older pch-split platforms. But this needs testing.
2333 */
2334 u32 pch_iir = I915_READ(SDEIIR);
Daniel Vetter92d03a82013-11-07 11:05:43 +01002335 if (pch_iir) {
2336 I915_WRITE(SDEIIR, pch_iir);
2337 ret = IRQ_HANDLED;
Oscar Mateo38cc46d2014-06-16 16:10:59 +01002338 cpt_irq_handler(dev, pch_iir);
2339 } else
2340 DRM_ERROR("The master control interrupt lied (SDE)!\n");
2341
Daniel Vetter92d03a82013-11-07 11:05:43 +01002342 }
2343
Ben Widawskyabd58f02013-11-02 21:07:09 -07002344 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
2345 POSTING_READ(GEN8_MASTER_IRQ);
2346
2347 return ret;
2348}
2349
Daniel Vetter17e1df02013-09-08 21:57:13 +02002350static void i915_error_wake_up(struct drm_i915_private *dev_priv,
2351 bool reset_completed)
2352{
Oscar Mateoa4872ba2014-05-22 14:13:33 +01002353 struct intel_engine_cs *ring;
Daniel Vetter17e1df02013-09-08 21:57:13 +02002354 int i;
2355
2356 /*
2357 * Notify all waiters for GPU completion events that reset state has
2358 * been changed, and that they need to restart their wait after
2359 * checking for potential errors (and bail out to drop locks if there is
2360 * a gpu reset pending so that i915_error_work_func can acquire them).
2361 */
2362
2363 /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
2364 for_each_ring(ring, dev_priv, i)
2365 wake_up_all(&ring->irq_queue);
2366
2367 /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
2368 wake_up_all(&dev_priv->pending_flip_queue);
2369
2370 /*
2371 * Signal tasks blocked in i915_gem_wait_for_error that the pending
2372 * reset state is cleared.
2373 */
2374 if (reset_completed)
2375 wake_up_all(&dev_priv->gpu_error.reset_queue);
2376}
2377
Jesse Barnes8a905232009-07-11 16:48:03 -04002378/**
2379 * i915_error_work_func - do process context error handling work
2380 * @work: work struct
2381 *
2382 * Fire an error uevent so userspace can see that a hang or error
2383 * was detected.
2384 */
2385static void i915_error_work_func(struct work_struct *work)
2386{
Daniel Vetter1f83fee2012-11-15 17:17:22 +01002387 struct i915_gpu_error *error = container_of(work, struct i915_gpu_error,
2388 work);
Jani Nikula2d1013d2014-03-31 14:27:17 +03002389 struct drm_i915_private *dev_priv =
2390 container_of(error, struct drm_i915_private, gpu_error);
Jesse Barnes8a905232009-07-11 16:48:03 -04002391 struct drm_device *dev = dev_priv->dev;
Ben Widawskycce723e2013-07-19 09:16:42 -07002392 char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
2393 char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
2394 char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
Daniel Vetter17e1df02013-09-08 21:57:13 +02002395 int ret;
Jesse Barnes8a905232009-07-11 16:48:03 -04002396
Dave Airlie5bdebb12013-10-11 14:07:25 +10002397 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
Jesse Barnes8a905232009-07-11 16:48:03 -04002398
Daniel Vetter7db0ba22012-12-06 16:23:37 +01002399 /*
2400 * Note that there's only one work item which does gpu resets, so we
2401 * need not worry about concurrent gpu resets potentially incrementing
2402 * error->reset_counter twice. We only need to take care of another
2403 * racing irq/hangcheck declaring the gpu dead for a second time. A
2404 * quick check for that is good enough: schedule_work ensures the
2405 * correct ordering between hang detection and this work item, and since
2406 * the reset in-progress bit is only ever set by code outside of this
2407 * work we don't need to worry about any other races.
2408 */
2409 if (i915_reset_in_progress(error) && !i915_terminally_wedged(error)) {
Chris Wilsonf803aa52010-09-19 12:38:26 +01002410 DRM_DEBUG_DRIVER("resetting chip\n");
Dave Airlie5bdebb12013-10-11 14:07:25 +10002411 kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
Daniel Vetter7db0ba22012-12-06 16:23:37 +01002412 reset_event);
Daniel Vetter1f83fee2012-11-15 17:17:22 +01002413
Daniel Vetter17e1df02013-09-08 21:57:13 +02002414 /*
Imre Deakf454c692014-04-23 01:09:04 +03002415 * In most cases it's guaranteed that we get here with an RPM
2416 * reference held, for example because there is a pending GPU
2417 * request that won't finish until the reset is done. This
2418 * isn't the case at least when we get here by doing a
2419 * simulated reset via debugs, so get an RPM reference.
2420 */
2421 intel_runtime_pm_get(dev_priv);
2422 /*
Daniel Vetter17e1df02013-09-08 21:57:13 +02002423 * All state reset _must_ be completed before we update the
2424 * reset counter, for otherwise waiters might miss the reset
2425 * pending state and not properly drop locks, resulting in
2426 * deadlocks with the reset work.
2427 */
Daniel Vetterf69061b2012-12-06 09:01:42 +01002428 ret = i915_reset(dev);
2429
Daniel Vetter17e1df02013-09-08 21:57:13 +02002430 intel_display_handle_reset(dev);
2431
Imre Deakf454c692014-04-23 01:09:04 +03002432 intel_runtime_pm_put(dev_priv);
2433
Daniel Vetterf69061b2012-12-06 09:01:42 +01002434 if (ret == 0) {
2435 /*
2436 * After all the gem state is reset, increment the reset
2437 * counter and wake up everyone waiting for the reset to
2438 * complete.
2439 *
2440 * Since unlock operations are a one-sided barrier only,
2441 * we need to insert a barrier here to order any seqno
2442 * updates before
2443 * the counter increment.
2444 */
Peter Zijlstra4e857c52014-03-17 18:06:10 +01002445 smp_mb__before_atomic();
Daniel Vetterf69061b2012-12-06 09:01:42 +01002446 atomic_inc(&dev_priv->gpu_error.reset_counter);
2447
Dave Airlie5bdebb12013-10-11 14:07:25 +10002448 kobject_uevent_env(&dev->primary->kdev->kobj,
Daniel Vetterf69061b2012-12-06 09:01:42 +01002449 KOBJ_CHANGE, reset_done_event);
Daniel Vetter1f83fee2012-11-15 17:17:22 +01002450 } else {
Mika Kuoppala2ac0f452013-11-12 14:44:19 +02002451 atomic_set_mask(I915_WEDGED, &error->reset_counter);
Ben Gamarif316a422009-09-14 17:48:46 -04002452 }
Daniel Vetter1f83fee2012-11-15 17:17:22 +01002453
Daniel Vetter17e1df02013-09-08 21:57:13 +02002454 /*
2455 * Note: The wake_up also serves as a memory barrier so that
2456 * waiters see the update value of the reset counter atomic_t.
2457 */
2458 i915_error_wake_up(dev_priv, true);
Ben Gamarif316a422009-09-14 17:48:46 -04002459 }
Jesse Barnes8a905232009-07-11 16:48:03 -04002460}
2461
Chris Wilson35aed2e2010-05-27 13:18:12 +01002462static void i915_report_and_clear_eir(struct drm_device *dev)
Jesse Barnes8a905232009-07-11 16:48:03 -04002463{
2464 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawskybd9854f2012-08-23 15:18:09 -07002465 uint32_t instdone[I915_NUM_INSTDONE_REG];
Jesse Barnes8a905232009-07-11 16:48:03 -04002466 u32 eir = I915_READ(EIR);
Ben Widawsky050ee912012-08-22 11:32:15 -07002467 int pipe, i;
Jesse Barnes8a905232009-07-11 16:48:03 -04002468
Chris Wilson35aed2e2010-05-27 13:18:12 +01002469 if (!eir)
2470 return;
Jesse Barnes8a905232009-07-11 16:48:03 -04002471
Joe Perchesa70491c2012-03-18 13:00:11 -07002472 pr_err("render error detected, EIR: 0x%08x\n", eir);
Jesse Barnes8a905232009-07-11 16:48:03 -04002473
Ben Widawskybd9854f2012-08-23 15:18:09 -07002474 i915_get_extra_instdone(dev, instdone);
2475
Jesse Barnes8a905232009-07-11 16:48:03 -04002476 if (IS_G4X(dev)) {
2477 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
2478 u32 ipeir = I915_READ(IPEIR_I965);
2479
Joe Perchesa70491c2012-03-18 13:00:11 -07002480 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2481 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
Ben Widawsky050ee912012-08-22 11:32:15 -07002482 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2483 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
Joe Perchesa70491c2012-03-18 13:00:11 -07002484 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
Joe Perchesa70491c2012-03-18 13:00:11 -07002485 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
Jesse Barnes8a905232009-07-11 16:48:03 -04002486 I915_WRITE(IPEIR_I965, ipeir);
Chris Wilson3143a2b2010-11-16 15:55:10 +00002487 POSTING_READ(IPEIR_I965);
Jesse Barnes8a905232009-07-11 16:48:03 -04002488 }
2489 if (eir & GM45_ERROR_PAGE_TABLE) {
2490 u32 pgtbl_err = I915_READ(PGTBL_ER);
Joe Perchesa70491c2012-03-18 13:00:11 -07002491 pr_err("page table error\n");
2492 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
Jesse Barnes8a905232009-07-11 16:48:03 -04002493 I915_WRITE(PGTBL_ER, pgtbl_err);
Chris Wilson3143a2b2010-11-16 15:55:10 +00002494 POSTING_READ(PGTBL_ER);
Jesse Barnes8a905232009-07-11 16:48:03 -04002495 }
2496 }
2497
Chris Wilsona6c45cf2010-09-17 00:32:17 +01002498 if (!IS_GEN2(dev)) {
Jesse Barnes8a905232009-07-11 16:48:03 -04002499 if (eir & I915_ERROR_PAGE_TABLE) {
2500 u32 pgtbl_err = I915_READ(PGTBL_ER);
Joe Perchesa70491c2012-03-18 13:00:11 -07002501 pr_err("page table error\n");
2502 pr_err(" PGTBL_ER: 0x%08x\n", pgtbl_err);
Jesse Barnes8a905232009-07-11 16:48:03 -04002503 I915_WRITE(PGTBL_ER, pgtbl_err);
Chris Wilson3143a2b2010-11-16 15:55:10 +00002504 POSTING_READ(PGTBL_ER);
Jesse Barnes8a905232009-07-11 16:48:03 -04002505 }
2506 }
2507
2508 if (eir & I915_ERROR_MEMORY_REFRESH) {
Joe Perchesa70491c2012-03-18 13:00:11 -07002509 pr_err("memory refresh error:\n");
Damien Lespiau055e3932014-08-18 13:49:10 +01002510 for_each_pipe(dev_priv, pipe)
Joe Perchesa70491c2012-03-18 13:00:11 -07002511 pr_err("pipe %c stat: 0x%08x\n",
Jesse Barnes9db4a9c2011-02-07 12:26:52 -08002512 pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
Jesse Barnes8a905232009-07-11 16:48:03 -04002513 /* pipestat has already been acked */
2514 }
2515 if (eir & I915_ERROR_INSTRUCTION) {
Joe Perchesa70491c2012-03-18 13:00:11 -07002516 pr_err("instruction error\n");
2517 pr_err(" INSTPM: 0x%08x\n", I915_READ(INSTPM));
Ben Widawsky050ee912012-08-22 11:32:15 -07002518 for (i = 0; i < ARRAY_SIZE(instdone); i++)
2519 pr_err(" INSTDONE_%d: 0x%08x\n", i, instdone[i]);
Chris Wilsona6c45cf2010-09-17 00:32:17 +01002520 if (INTEL_INFO(dev)->gen < 4) {
Jesse Barnes8a905232009-07-11 16:48:03 -04002521 u32 ipeir = I915_READ(IPEIR);
2522
Joe Perchesa70491c2012-03-18 13:00:11 -07002523 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR));
2524 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR));
Joe Perchesa70491c2012-03-18 13:00:11 -07002525 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD));
Jesse Barnes8a905232009-07-11 16:48:03 -04002526 I915_WRITE(IPEIR, ipeir);
Chris Wilson3143a2b2010-11-16 15:55:10 +00002527 POSTING_READ(IPEIR);
Jesse Barnes8a905232009-07-11 16:48:03 -04002528 } else {
2529 u32 ipeir = I915_READ(IPEIR_I965);
2530
Joe Perchesa70491c2012-03-18 13:00:11 -07002531 pr_err(" IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
2532 pr_err(" IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
Joe Perchesa70491c2012-03-18 13:00:11 -07002533 pr_err(" INSTPS: 0x%08x\n", I915_READ(INSTPS));
Joe Perchesa70491c2012-03-18 13:00:11 -07002534 pr_err(" ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
Jesse Barnes8a905232009-07-11 16:48:03 -04002535 I915_WRITE(IPEIR_I965, ipeir);
Chris Wilson3143a2b2010-11-16 15:55:10 +00002536 POSTING_READ(IPEIR_I965);
Jesse Barnes8a905232009-07-11 16:48:03 -04002537 }
2538 }
2539
2540 I915_WRITE(EIR, eir);
Chris Wilson3143a2b2010-11-16 15:55:10 +00002541 POSTING_READ(EIR);
Jesse Barnes8a905232009-07-11 16:48:03 -04002542 eir = I915_READ(EIR);
2543 if (eir) {
2544 /*
2545 * some errors might have become stuck,
2546 * mask them.
2547 */
2548 DRM_ERROR("EIR stuck: 0x%08x, masking\n", eir);
2549 I915_WRITE(EMR, I915_READ(EMR) | eir);
2550 I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
2551 }
Chris Wilson35aed2e2010-05-27 13:18:12 +01002552}
2553
2554/**
2555 * i915_handle_error - handle an error interrupt
2556 * @dev: drm device
2557 *
2558 * Do some basic checking of regsiter state at error interrupt time and
2559 * dump it to the syslog. Also call i915_capture_error_state() to make
2560 * sure we get a record and make it available in debugfs. Fire a uevent
2561 * so userspace knows something bad happened (should trigger collection
2562 * of a ring dump etc.).
2563 */
Mika Kuoppala58174462014-02-25 17:11:26 +02002564void i915_handle_error(struct drm_device *dev, bool wedged,
2565 const char *fmt, ...)
Chris Wilson35aed2e2010-05-27 13:18:12 +01002566{
2567 struct drm_i915_private *dev_priv = dev->dev_private;
Mika Kuoppala58174462014-02-25 17:11:26 +02002568 va_list args;
2569 char error_msg[80];
Chris Wilson35aed2e2010-05-27 13:18:12 +01002570
Mika Kuoppala58174462014-02-25 17:11:26 +02002571 va_start(args, fmt);
2572 vscnprintf(error_msg, sizeof(error_msg), fmt, args);
2573 va_end(args);
2574
2575 i915_capture_error_state(dev, wedged, error_msg);
Chris Wilson35aed2e2010-05-27 13:18:12 +01002576 i915_report_and_clear_eir(dev);
Jesse Barnes8a905232009-07-11 16:48:03 -04002577
Ben Gamariba1234d2009-09-14 17:48:47 -04002578 if (wedged) {
Daniel Vetterf69061b2012-12-06 09:01:42 +01002579 atomic_set_mask(I915_RESET_IN_PROGRESS_FLAG,
2580 &dev_priv->gpu_error.reset_counter);
Ben Gamariba1234d2009-09-14 17:48:47 -04002581
Ben Gamari11ed50e2009-09-14 17:48:45 -04002582 /*
Daniel Vetter17e1df02013-09-08 21:57:13 +02002583 * Wakeup waiting processes so that the reset work function
2584 * i915_error_work_func doesn't deadlock trying to grab various
2585 * locks. By bumping the reset counter first, the woken
2586 * processes will see a reset in progress and back off,
2587 * releasing their locks and then wait for the reset completion.
2588 * We must do this for _all_ gpu waiters that might hold locks
2589 * that the reset work needs to acquire.
2590 *
2591 * Note: The wake_up serves as the required memory barrier to
2592 * ensure that the waiters see the updated value of the reset
2593 * counter atomic_t.
Ben Gamari11ed50e2009-09-14 17:48:45 -04002594 */
Daniel Vetter17e1df02013-09-08 21:57:13 +02002595 i915_error_wake_up(dev_priv, false);
Ben Gamari11ed50e2009-09-14 17:48:45 -04002596 }
2597
Daniel Vetter122f46b2013-09-04 17:36:14 +02002598 /*
2599 * Our reset work can grab modeset locks (since it needs to reset the
2600 * state of outstanding pagelips). Hence it must not be run on our own
2601 * dev-priv->wq work queue for otherwise the flush_work in the pageflip
2602 * code will deadlock.
2603 */
2604 schedule_work(&dev_priv->gpu_error.work);
Jesse Barnes8a905232009-07-11 16:48:03 -04002605}
2606
Keith Packard42f52ef2008-10-18 19:39:29 -07002607/* Called from drm generic code, passed 'crtc' which
2608 * we use as a pipe index
2609 */
Jesse Barnesf71d4af2011-06-28 13:00:41 -07002610static int i915_enable_vblank(struct drm_device *dev, int pipe)
Jesse Barnes0a3e67a2008-09-30 12:14:26 -07002611{
Jani Nikula2d1013d2014-03-31 14:27:17 +03002612 struct drm_i915_private *dev_priv = dev->dev_private;
Keith Packarde9d21d72008-10-16 11:31:38 -07002613 unsigned long irqflags;
Jesse Barnes71e0ffa2009-01-08 10:42:15 -08002614
Chris Wilson5eddb702010-09-11 13:48:45 +01002615 if (!i915_pipe_enabled(dev, pipe))
Jesse Barnes71e0ffa2009-01-08 10:42:15 -08002616 return -EINVAL;
Jesse Barnes0a3e67a2008-09-30 12:14:26 -07002617
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002618 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Jesse Barnesf796cf82011-04-07 13:58:17 -07002619 if (INTEL_INFO(dev)->gen >= 4)
Keith Packard7c463582008-11-04 02:03:27 -08002620 i915_enable_pipestat(dev_priv, pipe,
Imre Deak755e9012014-02-10 18:42:47 +02002621 PIPE_START_VBLANK_INTERRUPT_STATUS);
Keith Packarde9d21d72008-10-16 11:31:38 -07002622 else
Keith Packard7c463582008-11-04 02:03:27 -08002623 i915_enable_pipestat(dev_priv, pipe,
Imre Deak755e9012014-02-10 18:42:47 +02002624 PIPE_VBLANK_INTERRUPT_STATUS);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002625 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
Chris Wilson8692d00e2011-02-05 10:08:21 +00002626
Jesse Barnes0a3e67a2008-09-30 12:14:26 -07002627 return 0;
2628}
2629
Jesse Barnesf71d4af2011-06-28 13:00:41 -07002630static int ironlake_enable_vblank(struct drm_device *dev, int pipe)
Jesse Barnesf796cf82011-04-07 13:58:17 -07002631{
Jani Nikula2d1013d2014-03-31 14:27:17 +03002632 struct drm_i915_private *dev_priv = dev->dev_private;
Jesse Barnesf796cf82011-04-07 13:58:17 -07002633 unsigned long irqflags;
Paulo Zanonib5184212013-07-12 20:00:08 -03002634 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
Daniel Vetter40da17c2013-10-21 18:04:36 +02002635 DE_PIPE_VBLANK(pipe);
Jesse Barnesf796cf82011-04-07 13:58:17 -07002636
2637 if (!i915_pipe_enabled(dev, pipe))
2638 return -EINVAL;
2639
2640 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Paulo Zanonib5184212013-07-12 20:00:08 -03002641 ironlake_enable_display_irq(dev_priv, bit);
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002642 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2643
2644 return 0;
2645}
2646
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002647static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
2648{
Jani Nikula2d1013d2014-03-31 14:27:17 +03002649 struct drm_i915_private *dev_priv = dev->dev_private;
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002650 unsigned long irqflags;
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002651
2652 if (!i915_pipe_enabled(dev, pipe))
2653 return -EINVAL;
2654
2655 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Jesse Barnes31acc7f2012-06-20 10:53:11 -07002656 i915_enable_pipestat(dev_priv, pipe,
Imre Deak755e9012014-02-10 18:42:47 +02002657 PIPE_START_VBLANK_INTERRUPT_STATUS);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002658 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2659
2660 return 0;
2661}
2662
Ben Widawskyabd58f02013-11-02 21:07:09 -07002663static int gen8_enable_vblank(struct drm_device *dev, int pipe)
2664{
2665 struct drm_i915_private *dev_priv = dev->dev_private;
2666 unsigned long irqflags;
Ben Widawskyabd58f02013-11-02 21:07:09 -07002667
2668 if (!i915_pipe_enabled(dev, pipe))
2669 return -EINVAL;
2670
2671 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Daniel Vetter7167d7c2013-11-07 11:05:45 +01002672 dev_priv->de_irq_mask[pipe] &= ~GEN8_PIPE_VBLANK;
2673 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2674 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
Ben Widawskyabd58f02013-11-02 21:07:09 -07002675 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2676 return 0;
2677}
2678
Keith Packard42f52ef2008-10-18 19:39:29 -07002679/* Called from drm generic code, passed 'crtc' which
2680 * we use as a pipe index
2681 */
Jesse Barnesf71d4af2011-06-28 13:00:41 -07002682static void i915_disable_vblank(struct drm_device *dev, int pipe)
Jesse Barnes0a3e67a2008-09-30 12:14:26 -07002683{
Jani Nikula2d1013d2014-03-31 14:27:17 +03002684 struct drm_i915_private *dev_priv = dev->dev_private;
Keith Packarde9d21d72008-10-16 11:31:38 -07002685 unsigned long irqflags;
Jesse Barnes0a3e67a2008-09-30 12:14:26 -07002686
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002687 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Jesse Barnesf796cf82011-04-07 13:58:17 -07002688 i915_disable_pipestat(dev_priv, pipe,
Imre Deak755e9012014-02-10 18:42:47 +02002689 PIPE_VBLANK_INTERRUPT_STATUS |
2690 PIPE_START_VBLANK_INTERRUPT_STATUS);
Jesse Barnesf796cf82011-04-07 13:58:17 -07002691 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2692}
2693
Jesse Barnesf71d4af2011-06-28 13:00:41 -07002694static void ironlake_disable_vblank(struct drm_device *dev, int pipe)
Jesse Barnesf796cf82011-04-07 13:58:17 -07002695{
Jani Nikula2d1013d2014-03-31 14:27:17 +03002696 struct drm_i915_private *dev_priv = dev->dev_private;
Jesse Barnesf796cf82011-04-07 13:58:17 -07002697 unsigned long irqflags;
Paulo Zanonib5184212013-07-12 20:00:08 -03002698 uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
Daniel Vetter40da17c2013-10-21 18:04:36 +02002699 DE_PIPE_VBLANK(pipe);
Jesse Barnesf796cf82011-04-07 13:58:17 -07002700
2701 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Paulo Zanonib5184212013-07-12 20:00:08 -03002702 ironlake_disable_display_irq(dev_priv, bit);
Jesse Barnesb1f14ad2011-04-06 12:13:38 -07002703 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2704}
2705
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002706static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
2707{
Jani Nikula2d1013d2014-03-31 14:27:17 +03002708 struct drm_i915_private *dev_priv = dev->dev_private;
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002709 unsigned long irqflags;
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002710
2711 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Jesse Barnes31acc7f2012-06-20 10:53:11 -07002712 i915_disable_pipestat(dev_priv, pipe,
Imre Deak755e9012014-02-10 18:42:47 +02002713 PIPE_START_VBLANK_INTERRUPT_STATUS);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07002714 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2715}
2716
Ben Widawskyabd58f02013-11-02 21:07:09 -07002717static void gen8_disable_vblank(struct drm_device *dev, int pipe)
2718{
2719 struct drm_i915_private *dev_priv = dev->dev_private;
2720 unsigned long irqflags;
Ben Widawskyabd58f02013-11-02 21:07:09 -07002721
2722 if (!i915_pipe_enabled(dev, pipe))
2723 return;
2724
2725 spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
Daniel Vetter7167d7c2013-11-07 11:05:45 +01002726 dev_priv->de_irq_mask[pipe] |= GEN8_PIPE_VBLANK;
2727 I915_WRITE(GEN8_DE_PIPE_IMR(pipe), dev_priv->de_irq_mask[pipe]);
2728 POSTING_READ(GEN8_DE_PIPE_IMR(pipe));
Ben Widawskyabd58f02013-11-02 21:07:09 -07002729 spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
2730}
2731
Chris Wilson893eead2010-10-27 14:44:35 +01002732static u32
Oscar Mateoa4872ba2014-05-22 14:13:33 +01002733ring_last_seqno(struct intel_engine_cs *ring)
Zou Nan hai852835f2010-05-21 09:08:56 +08002734{
Chris Wilson893eead2010-10-27 14:44:35 +01002735 return list_entry(ring->request_list.prev,
2736 struct drm_i915_gem_request, list)->seqno;
2737}
2738
Chris Wilson9107e9d2013-06-10 11:20:20 +01002739static bool
Oscar Mateoa4872ba2014-05-22 14:13:33 +01002740ring_idle(struct intel_engine_cs *ring, u32 seqno)
Chris Wilson893eead2010-10-27 14:44:35 +01002741{
Chris Wilson9107e9d2013-06-10 11:20:20 +01002742 return (list_empty(&ring->request_list) ||
2743 i915_seqno_passed(seqno, ring_last_seqno(ring)));
Ben Gamarif65d9422009-09-14 17:48:44 -04002744}
2745
Daniel Vettera028c4b2014-03-15 00:08:56 +01002746static bool
2747ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
2748{
2749 if (INTEL_INFO(dev)->gen >= 8) {
Rodrigo Vivia6cdb932014-06-30 09:53:39 -07002750 return (ipehr >> 23) == 0x1c;
Daniel Vettera028c4b2014-03-15 00:08:56 +01002751 } else {
2752 ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
2753 return ipehr == (MI_SEMAPHORE_MBOX | MI_SEMAPHORE_COMPARE |
2754 MI_SEMAPHORE_REGISTER);
2755 }
2756}
2757
Oscar Mateoa4872ba2014-05-22 14:13:33 +01002758static struct intel_engine_cs *
Rodrigo Vivia6cdb932014-06-30 09:53:39 -07002759semaphore_wait_to_signaller_ring(struct intel_engine_cs *ring, u32 ipehr, u64 offset)
Daniel Vetter921d42e2014-03-18 10:26:04 +01002760{
2761 struct drm_i915_private *dev_priv = ring->dev->dev_private;
Oscar Mateoa4872ba2014-05-22 14:13:33 +01002762 struct intel_engine_cs *signaller;
Daniel Vetter921d42e2014-03-18 10:26:04 +01002763 int i;
2764
2765 if (INTEL_INFO(dev_priv->dev)->gen >= 8) {
Rodrigo Vivia6cdb932014-06-30 09:53:39 -07002766 for_each_ring(signaller, dev_priv, i) {
2767 if (ring == signaller)
2768 continue;
2769
2770 if (offset == signaller->semaphore.signal_ggtt[ring->id])
2771 return signaller;
2772 }
Daniel Vetter921d42e2014-03-18 10:26:04 +01002773 } else {
2774 u32 sync_bits = ipehr & MI_SEMAPHORE_SYNC_MASK;
2775
2776 for_each_ring(signaller, dev_priv, i) {
2777 if(ring == signaller)
2778 continue;
2779
Ben Widawskyebc348b2014-04-29 14:52:28 -07002780 if (sync_bits == signaller->semaphore.mbox.wait[ring->id])
Daniel Vetter921d42e2014-03-18 10:26:04 +01002781 return signaller;
2782 }
2783 }
2784
Rodrigo Vivia6cdb932014-06-30 09:53:39 -07002785 DRM_ERROR("No signaller ring found for ring %i, ipehr 0x%08x, offset 0x%016llx\n",
2786 ring->id, ipehr, offset);
Daniel Vetter921d42e2014-03-18 10:26:04 +01002787
2788 return NULL;
2789}
2790
Oscar Mateoa4872ba2014-05-22 14:13:33 +01002791static struct intel_engine_cs *
2792semaphore_waits_for(struct intel_engine_cs *ring, u32 *seqno)
Chris Wilsona24a11e2013-03-14 17:52:05 +02002793{
2794 struct drm_i915_private *dev_priv = ring->dev->dev_private;
Daniel Vetter88fe4292014-03-15 00:08:55 +01002795 u32 cmd, ipehr, head;
Rodrigo Vivia6cdb932014-06-30 09:53:39 -07002796 u64 offset = 0;
2797 int i, backwards;
Chris Wilsona24a11e2013-03-14 17:52:05 +02002798
2799 ipehr = I915_READ(RING_IPEHR(ring->mmio_base));
Daniel Vettera028c4b2014-03-15 00:08:56 +01002800 if (!ipehr_is_semaphore_wait(ring->dev, ipehr))
Chris Wilson6274f212013-06-10 11:20:21 +01002801 return NULL;
Chris Wilsona24a11e2013-03-14 17:52:05 +02002802
Daniel Vetter88fe4292014-03-15 00:08:55 +01002803 /*
2804 * HEAD is likely pointing to the dword after the actual command,
2805 * so scan backwards until we find the MBOX. But limit it to just 3
Rodrigo Vivia6cdb932014-06-30 09:53:39 -07002806 * or 4 dwords depending on the semaphore wait command size.
2807 * Note that we don't care about ACTHD here since that might
Daniel Vetter88fe4292014-03-15 00:08:55 +01002808 * point at at batch, and semaphores are always emitted into the
2809 * ringbuffer itself.
Chris Wilsona24a11e2013-03-14 17:52:05 +02002810 */
Daniel Vetter88fe4292014-03-15 00:08:55 +01002811 head = I915_READ_HEAD(ring) & HEAD_ADDR;
Rodrigo Vivia6cdb932014-06-30 09:53:39 -07002812 backwards = (INTEL_INFO(ring->dev)->gen >= 8) ? 5 : 4;
Daniel Vetter88fe4292014-03-15 00:08:55 +01002813
Rodrigo Vivia6cdb932014-06-30 09:53:39 -07002814 for (i = backwards; i; --i) {
Daniel Vetter88fe4292014-03-15 00:08:55 +01002815 /*
2816 * Be paranoid and presume the hw has gone off into the wild -
2817 * our ring is smaller than what the hardware (and hence
2818 * HEAD_ADDR) allows. Also handles wrap-around.
2819 */
Oscar Mateoee1b1e52014-05-22 14:13:35 +01002820 head &= ring->buffer->size - 1;
Daniel Vetter88fe4292014-03-15 00:08:55 +01002821
2822 /* This here seems to blow up */
Oscar Mateoee1b1e52014-05-22 14:13:35 +01002823 cmd = ioread32(ring->buffer->virtual_start + head);
Chris Wilsona24a11e2013-03-14 17:52:05 +02002824 if (cmd == ipehr)
2825 break;
2826
Daniel Vetter88fe4292014-03-15 00:08:55 +01002827 head -= 4;
2828 }
Chris Wilsona24a11e2013-03-14 17:52:05 +02002829
Daniel Vetter88fe4292014-03-15 00:08:55 +01002830 if (!i)
2831 return NULL;
2832
Oscar Mateoee1b1e52014-05-22 14:13:35 +01002833 *seqno = ioread32(ring->buffer->virtual_start + head + 4) + 1;
Rodrigo Vivia6cdb932014-06-30 09:53:39 -07002834 if (INTEL_INFO(ring->dev)->gen >= 8) {
2835 offset = ioread32(ring->buffer->virtual_start + head + 12);
2836 offset <<= 32;
2837 offset = ioread32(ring->buffer->virtual_start + head + 8);
2838 }
2839 return semaphore_wait_to_signaller_ring(ring, ipehr, offset);
Chris Wilsona24a11e2013-03-14 17:52:05 +02002840}
2841
Oscar Mateoa4872ba2014-05-22 14:13:33 +01002842static int semaphore_passed(struct intel_engine_cs *ring)
Chris Wilson6274f212013-06-10 11:20:21 +01002843{
2844 struct drm_i915_private *dev_priv = ring->dev->dev_private;
Oscar Mateoa4872ba2014-05-22 14:13:33 +01002845 struct intel_engine_cs *signaller;
Chris Wilsona0d036b2014-07-19 12:40:42 +01002846 u32 seqno;
Chris Wilson6274f212013-06-10 11:20:21 +01002847
Chris Wilson4be17382014-06-06 10:22:29 +01002848 ring->hangcheck.deadlock++;
Chris Wilson6274f212013-06-10 11:20:21 +01002849
2850 signaller = semaphore_waits_for(ring, &seqno);
Chris Wilson4be17382014-06-06 10:22:29 +01002851 if (signaller == NULL)
2852 return -1;
2853
2854 /* Prevent pathological recursion due to driver bugs */
2855 if (signaller->hangcheck.deadlock >= I915_NUM_RINGS)
Chris Wilson6274f212013-06-10 11:20:21 +01002856 return -1;
2857
Chris Wilson4be17382014-06-06 10:22:29 +01002858 if (i915_seqno_passed(signaller->get_seqno(signaller, false), seqno))
2859 return 1;
2860
Chris Wilsona0d036b2014-07-19 12:40:42 +01002861 /* cursory check for an unkickable deadlock */
2862 if (I915_READ_CTL(signaller) & RING_WAIT_SEMAPHORE &&
2863 semaphore_passed(signaller) < 0)
Chris Wilson4be17382014-06-06 10:22:29 +01002864 return -1;
2865
2866 return 0;
Chris Wilson6274f212013-06-10 11:20:21 +01002867}
2868
2869static void semaphore_clear_deadlocks(struct drm_i915_private *dev_priv)
2870{
Oscar Mateoa4872ba2014-05-22 14:13:33 +01002871 struct intel_engine_cs *ring;
Chris Wilson6274f212013-06-10 11:20:21 +01002872 int i;
2873
2874 for_each_ring(ring, dev_priv, i)
Chris Wilson4be17382014-06-06 10:22:29 +01002875 ring->hangcheck.deadlock = 0;
Chris Wilson6274f212013-06-10 11:20:21 +01002876}
2877
Mika Kuoppalaad8beae2013-06-12 12:35:32 +03002878static enum intel_ring_hangcheck_action
Oscar Mateoa4872ba2014-05-22 14:13:33 +01002879ring_stuck(struct intel_engine_cs *ring, u64 acthd)
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002880{
2881 struct drm_device *dev = ring->dev;
2882 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson9107e9d2013-06-10 11:20:20 +01002883 u32 tmp;
2884
Mika Kuoppalaf260fe72014-08-05 17:16:26 +03002885 if (acthd != ring->hangcheck.acthd) {
2886 if (acthd > ring->hangcheck.max_acthd) {
2887 ring->hangcheck.max_acthd = acthd;
2888 return HANGCHECK_ACTIVE;
2889 }
2890
2891 return HANGCHECK_ACTIVE_LOOP;
2892 }
Chris Wilson6274f212013-06-10 11:20:21 +01002893
Chris Wilson9107e9d2013-06-10 11:20:20 +01002894 if (IS_GEN2(dev))
Jani Nikulaf2f4d822013-08-11 12:44:01 +03002895 return HANGCHECK_HUNG;
Chris Wilson9107e9d2013-06-10 11:20:20 +01002896
2897 /* Is the chip hanging on a WAIT_FOR_EVENT?
2898 * If so we can simply poke the RB_WAIT bit
2899 * and break the hang. This should work on
2900 * all but the second generation chipsets.
2901 */
2902 tmp = I915_READ_CTL(ring);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002903 if (tmp & RING_WAIT) {
Mika Kuoppala58174462014-02-25 17:11:26 +02002904 i915_handle_error(dev, false,
2905 "Kicking stuck wait on %s",
2906 ring->name);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002907 I915_WRITE_CTL(ring, tmp);
Jani Nikulaf2f4d822013-08-11 12:44:01 +03002908 return HANGCHECK_KICK;
Chris Wilson1ec14ad2010-12-04 11:30:53 +00002909 }
Chris Wilsona24a11e2013-03-14 17:52:05 +02002910
Chris Wilson6274f212013-06-10 11:20:21 +01002911 if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
2912 switch (semaphore_passed(ring)) {
2913 default:
Jani Nikulaf2f4d822013-08-11 12:44:01 +03002914 return HANGCHECK_HUNG;
Chris Wilson6274f212013-06-10 11:20:21 +01002915 case 1:
Mika Kuoppala58174462014-02-25 17:11:26 +02002916 i915_handle_error(dev, false,
2917 "Kicking stuck semaphore on %s",
2918 ring->name);
Chris Wilson6274f212013-06-10 11:20:21 +01002919 I915_WRITE_CTL(ring, tmp);
Jani Nikulaf2f4d822013-08-11 12:44:01 +03002920 return HANGCHECK_KICK;
Chris Wilson6274f212013-06-10 11:20:21 +01002921 case 0:
Jani Nikulaf2f4d822013-08-11 12:44:01 +03002922 return HANGCHECK_WAIT;
Chris Wilson6274f212013-06-10 11:20:21 +01002923 }
Chris Wilson9107e9d2013-06-10 11:20:20 +01002924 }
Mika Kuoppalaed5cbb02013-05-13 16:32:11 +03002925
Jani Nikulaf2f4d822013-08-11 12:44:01 +03002926 return HANGCHECK_HUNG;
Mika Kuoppalaed5cbb02013-05-13 16:32:11 +03002927}
2928
Ben Gamarif65d9422009-09-14 17:48:44 -04002929/**
2930 * This is called when the chip hasn't reported back with completed
Mika Kuoppala05407ff2013-05-30 09:04:29 +03002931 * batchbuffers in a long time. We keep track per ring seqno progress and
2932 * if there are no progress, hangcheck score for that ring is increased.
2933 * Further, acthd is inspected to see if the ring is stuck. On stuck case
2934 * we kick the ring. If we see no progress on three subsequent calls
2935 * we assume chip is wedged and try to fix it by resetting the chip.
Ben Gamarif65d9422009-09-14 17:48:44 -04002936 */
Damien Lespiaua658b5d2013-08-08 22:28:56 +01002937static void i915_hangcheck_elapsed(unsigned long data)
Ben Gamarif65d9422009-09-14 17:48:44 -04002938{
2939 struct drm_device *dev = (struct drm_device *)data;
Jani Nikula2d1013d2014-03-31 14:27:17 +03002940 struct drm_i915_private *dev_priv = dev->dev_private;
Oscar Mateoa4872ba2014-05-22 14:13:33 +01002941 struct intel_engine_cs *ring;
Chris Wilsonb4519512012-05-11 14:29:30 +01002942 int i;
Mika Kuoppala05407ff2013-05-30 09:04:29 +03002943 int busy_count = 0, rings_hung = 0;
Chris Wilson9107e9d2013-06-10 11:20:20 +01002944 bool stuck[I915_NUM_RINGS] = { 0 };
2945#define BUSY 1
2946#define KICK 5
2947#define HUNG 20
Chris Wilson893eead2010-10-27 14:44:35 +01002948
Jani Nikulad330a952014-01-21 11:24:25 +02002949 if (!i915.enable_hangcheck)
Ben Widawsky3e0dc6b2011-06-29 10:26:42 -07002950 return;
2951
Chris Wilsonb4519512012-05-11 14:29:30 +01002952 for_each_ring(ring, dev_priv, i) {
Chris Wilson50877442014-03-21 12:41:53 +00002953 u64 acthd;
2954 u32 seqno;
Chris Wilson9107e9d2013-06-10 11:20:20 +01002955 bool busy = true;
Chris Wilsonb4519512012-05-11 14:29:30 +01002956
Chris Wilson6274f212013-06-10 11:20:21 +01002957 semaphore_clear_deadlocks(dev_priv);
2958
Mika Kuoppala05407ff2013-05-30 09:04:29 +03002959 seqno = ring->get_seqno(ring, false);
2960 acthd = intel_ring_get_active_head(ring);
Chris Wilsond1e61e72012-04-10 17:00:41 +01002961
Chris Wilson9107e9d2013-06-10 11:20:20 +01002962 if (ring->hangcheck.seqno == seqno) {
2963 if (ring_idle(ring, seqno)) {
Mika Kuoppalada661462013-09-06 16:03:28 +03002964 ring->hangcheck.action = HANGCHECK_IDLE;
2965
Chris Wilson9107e9d2013-06-10 11:20:20 +01002966 if (waitqueue_active(&ring->irq_queue)) {
2967 /* Issue a wake-up to catch stuck h/w. */
Chris Wilson094f9a52013-09-25 17:34:55 +01002968 if (!test_and_set_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings)) {
Daniel Vetterf4adcd22013-10-28 09:24:13 +01002969 if (!(dev_priv->gpu_error.test_irq_rings & intel_ring_flag(ring)))
2970 DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
2971 ring->name);
2972 else
2973 DRM_INFO("Fake missed irq on %s\n",
2974 ring->name);
Chris Wilson094f9a52013-09-25 17:34:55 +01002975 wake_up_all(&ring->irq_queue);
2976 }
2977 /* Safeguard against driver failure */
2978 ring->hangcheck.score += BUSY;
Chris Wilson9107e9d2013-06-10 11:20:20 +01002979 } else
2980 busy = false;
Mika Kuoppala05407ff2013-05-30 09:04:29 +03002981 } else {
Chris Wilson6274f212013-06-10 11:20:21 +01002982 /* We always increment the hangcheck score
2983 * if the ring is busy and still processing
2984 * the same request, so that no single request
2985 * can run indefinitely (such as a chain of
2986 * batches). The only time we do not increment
2987 * the hangcheck score on this ring, if this
2988 * ring is in a legitimate wait for another
2989 * ring. In that case the waiting ring is a
2990 * victim and we want to be sure we catch the
2991 * right culprit. Then every time we do kick
2992 * the ring, add a small increment to the
2993 * score so that we can catch a batch that is
2994 * being repeatedly kicked and so responsible
2995 * for stalling the machine.
2996 */
Mika Kuoppalaad8beae2013-06-12 12:35:32 +03002997 ring->hangcheck.action = ring_stuck(ring,
2998 acthd);
2999
3000 switch (ring->hangcheck.action) {
Mika Kuoppalada661462013-09-06 16:03:28 +03003001 case HANGCHECK_IDLE:
Jani Nikulaf2f4d822013-08-11 12:44:01 +03003002 case HANGCHECK_WAIT:
Jani Nikulaf2f4d822013-08-11 12:44:01 +03003003 case HANGCHECK_ACTIVE:
Mika Kuoppalaf260fe72014-08-05 17:16:26 +03003004 break;
3005 case HANGCHECK_ACTIVE_LOOP:
Jani Nikulaea04cb32013-08-11 12:44:02 +03003006 ring->hangcheck.score += BUSY;
Chris Wilson6274f212013-06-10 11:20:21 +01003007 break;
Jani Nikulaf2f4d822013-08-11 12:44:01 +03003008 case HANGCHECK_KICK:
Jani Nikulaea04cb32013-08-11 12:44:02 +03003009 ring->hangcheck.score += KICK;
Chris Wilson6274f212013-06-10 11:20:21 +01003010 break;
Jani Nikulaf2f4d822013-08-11 12:44:01 +03003011 case HANGCHECK_HUNG:
Jani Nikulaea04cb32013-08-11 12:44:02 +03003012 ring->hangcheck.score += HUNG;
Chris Wilson6274f212013-06-10 11:20:21 +01003013 stuck[i] = true;
3014 break;
3015 }
Mika Kuoppala05407ff2013-05-30 09:04:29 +03003016 }
Chris Wilson9107e9d2013-06-10 11:20:20 +01003017 } else {
Mika Kuoppalada661462013-09-06 16:03:28 +03003018 ring->hangcheck.action = HANGCHECK_ACTIVE;
3019
Chris Wilson9107e9d2013-06-10 11:20:20 +01003020 /* Gradually reduce the count so that we catch DoS
3021 * attempts across multiple batches.
3022 */
3023 if (ring->hangcheck.score > 0)
3024 ring->hangcheck.score--;
Mika Kuoppalaf260fe72014-08-05 17:16:26 +03003025
3026 ring->hangcheck.acthd = ring->hangcheck.max_acthd = 0;
Chris Wilsond1e61e72012-04-10 17:00:41 +01003027 }
3028
Mika Kuoppala05407ff2013-05-30 09:04:29 +03003029 ring->hangcheck.seqno = seqno;
3030 ring->hangcheck.acthd = acthd;
Chris Wilson9107e9d2013-06-10 11:20:20 +01003031 busy_count += busy;
Chris Wilson893eead2010-10-27 14:44:35 +01003032 }
Eric Anholtb9201c12010-01-08 14:25:16 -08003033
Mika Kuoppala92cab732013-05-24 17:16:07 +03003034 for_each_ring(ring, dev_priv, i) {
Mika Kuoppalab6b0fac2014-01-30 19:04:43 +02003035 if (ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
Daniel Vetterb8d88d12013-08-28 10:57:59 +02003036 DRM_INFO("%s on %s\n",
3037 stuck[i] ? "stuck" : "no progress",
3038 ring->name);
Chris Wilsona43adf02013-06-10 11:20:22 +01003039 rings_hung++;
Mika Kuoppala92cab732013-05-24 17:16:07 +03003040 }
3041 }
3042
Mika Kuoppala05407ff2013-05-30 09:04:29 +03003043 if (rings_hung)
Mika Kuoppala58174462014-02-25 17:11:26 +02003044 return i915_handle_error(dev, true, "Ring hung");
Ben Gamarif65d9422009-09-14 17:48:44 -04003045
Mika Kuoppala05407ff2013-05-30 09:04:29 +03003046 if (busy_count)
3047 /* Reset timer case chip hangs without another request
3048 * being added */
Mika Kuoppala10cd45b2013-07-03 17:22:08 +03003049 i915_queue_hangcheck(dev);
3050}
3051
3052void i915_queue_hangcheck(struct drm_device *dev)
3053{
3054 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson672e7b72014-11-19 09:47:19 +00003055 struct timer_list *timer = &dev_priv->gpu_error.hangcheck_timer;
3056
Jani Nikulad330a952014-01-21 11:24:25 +02003057 if (!i915.enable_hangcheck)
Mika Kuoppala10cd45b2013-07-03 17:22:08 +03003058 return;
3059
Chris Wilson672e7b72014-11-19 09:47:19 +00003060 /* Don't continually defer the hangcheck, but make sure it is active */
3061 if (!timer_pending(timer))
3062 timer->expires = round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES);
3063 mod_timer(timer, timer->expires);
Ben Gamarif65d9422009-09-14 17:48:44 -04003064}
3065
Paulo Zanoni1c69eb42014-04-01 15:37:23 -03003066static void ibx_irq_reset(struct drm_device *dev)
Paulo Zanoni91738a92013-06-05 14:21:51 -03003067{
3068 struct drm_i915_private *dev_priv = dev->dev_private;
3069
3070 if (HAS_PCH_NOP(dev))
3071 return;
3072
Paulo Zanonif86f3fb2014-04-01 15:37:14 -03003073 GEN5_IRQ_RESET(SDE);
Paulo Zanoni105b1222014-04-01 15:37:17 -03003074
3075 if (HAS_PCH_CPT(dev) || HAS_PCH_LPT(dev))
3076 I915_WRITE(SERR_INT, 0xffffffff);
Paulo Zanoni622364b2014-04-01 15:37:22 -03003077}
Paulo Zanoni105b1222014-04-01 15:37:17 -03003078
Paulo Zanoni622364b2014-04-01 15:37:22 -03003079/*
3080 * SDEIER is also touched by the interrupt handler to work around missed PCH
3081 * interrupts. Hence we can't update it after the interrupt handler is enabled -
3082 * instead we unconditionally enable all PCH interrupt sources here, but then
3083 * only unmask them as needed with SDEIMR.
3084 *
3085 * This function needs to be called before interrupts are enabled.
3086 */
3087static void ibx_irq_pre_postinstall(struct drm_device *dev)
3088{
3089 struct drm_i915_private *dev_priv = dev->dev_private;
3090
3091 if (HAS_PCH_NOP(dev))
3092 return;
3093
3094 WARN_ON(I915_READ(SDEIER) != 0);
Paulo Zanoni91738a92013-06-05 14:21:51 -03003095 I915_WRITE(SDEIER, 0xffffffff);
3096 POSTING_READ(SDEIER);
3097}
3098
Paulo Zanoni7c4d6642014-04-01 15:37:19 -03003099static void gen5_gt_irq_reset(struct drm_device *dev)
Daniel Vetterd18ea1b2013-07-12 22:43:25 +02003100{
3101 struct drm_i915_private *dev_priv = dev->dev_private;
3102
Paulo Zanonif86f3fb2014-04-01 15:37:14 -03003103 GEN5_IRQ_RESET(GT);
Paulo Zanonia9d356a2014-04-01 15:37:09 -03003104 if (INTEL_INFO(dev)->gen >= 6)
Paulo Zanonif86f3fb2014-04-01 15:37:14 -03003105 GEN5_IRQ_RESET(GEN6_PM);
Daniel Vetterd18ea1b2013-07-12 22:43:25 +02003106}
3107
Linus Torvalds1da177e2005-04-16 15:20:36 -07003108/* drm_dma.h hooks
3109*/
Paulo Zanonibe30b292014-04-01 15:37:25 -03003110static void ironlake_irq_reset(struct drm_device *dev)
Zhenyu Wang036a4a72009-06-08 14:40:19 +08003111{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003112 struct drm_i915_private *dev_priv = dev->dev_private;
Zhenyu Wang036a4a72009-06-08 14:40:19 +08003113
Paulo Zanoni0c841212014-04-01 15:37:27 -03003114 I915_WRITE(HWSTAM, 0xffffffff);
Daniel Vetterbdfcdb62012-01-05 01:05:26 +01003115
Paulo Zanonif86f3fb2014-04-01 15:37:14 -03003116 GEN5_IRQ_RESET(DE);
Paulo Zanonic6d954c2014-04-01 15:37:18 -03003117 if (IS_GEN7(dev))
3118 I915_WRITE(GEN7_ERR_INT, 0xffffffff);
Zhenyu Wang036a4a72009-06-08 14:40:19 +08003119
Paulo Zanoni7c4d6642014-04-01 15:37:19 -03003120 gen5_gt_irq_reset(dev);
Zhenyu Wangc6501562009-11-03 18:57:21 +00003121
Paulo Zanoni1c69eb42014-04-01 15:37:23 -03003122 ibx_irq_reset(dev);
Ben Widawsky7d991632013-05-28 19:22:25 -07003123}
3124
Ville Syrjälä70591a42014-10-30 19:42:58 +02003125static void vlv_display_irq_reset(struct drm_i915_private *dev_priv)
3126{
3127 enum pipe pipe;
3128
3129 I915_WRITE(PORT_HOTPLUG_EN, 0);
3130 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3131
3132 for_each_pipe(dev_priv, pipe)
3133 I915_WRITE(PIPESTAT(pipe), 0xffff);
3134
3135 GEN5_IRQ_RESET(VLV_);
3136}
3137
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003138static void valleyview_irq_preinstall(struct drm_device *dev)
3139{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003140 struct drm_i915_private *dev_priv = dev->dev_private;
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003141
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003142 /* VLV magic */
3143 I915_WRITE(VLV_IMR, 0);
3144 I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
3145 I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
3146 I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
3147
Paulo Zanoni7c4d6642014-04-01 15:37:19 -03003148 gen5_gt_irq_reset(dev);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003149
Ville Syrjälä7c4cde32014-10-30 19:42:51 +02003150 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003151
Ville Syrjälä70591a42014-10-30 19:42:58 +02003152 vlv_display_irq_reset(dev_priv);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003153}
3154
Daniel Vetterd6e3cca2014-05-22 22:18:22 +02003155static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
3156{
3157 GEN8_IRQ_RESET_NDX(GT, 0);
3158 GEN8_IRQ_RESET_NDX(GT, 1);
3159 GEN8_IRQ_RESET_NDX(GT, 2);
3160 GEN8_IRQ_RESET_NDX(GT, 3);
3161}
3162
Paulo Zanoni823f6b32014-04-01 15:37:26 -03003163static void gen8_irq_reset(struct drm_device *dev)
Ben Widawskyabd58f02013-11-02 21:07:09 -07003164{
3165 struct drm_i915_private *dev_priv = dev->dev_private;
3166 int pipe;
3167
Ben Widawskyabd58f02013-11-02 21:07:09 -07003168 I915_WRITE(GEN8_MASTER_IRQ, 0);
3169 POSTING_READ(GEN8_MASTER_IRQ);
3170
Daniel Vetterd6e3cca2014-05-22 22:18:22 +02003171 gen8_gt_irq_reset(dev_priv);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003172
Damien Lespiau055e3932014-08-18 13:49:10 +01003173 for_each_pipe(dev_priv, pipe)
Daniel Vetterf458ebb2014-09-30 10:56:39 +02003174 if (intel_display_power_is_enabled(dev_priv,
3175 POWER_DOMAIN_PIPE(pipe)))
Paulo Zanoni813bde42014-07-04 11:50:29 -03003176 GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003177
Paulo Zanonif86f3fb2014-04-01 15:37:14 -03003178 GEN5_IRQ_RESET(GEN8_DE_PORT_);
3179 GEN5_IRQ_RESET(GEN8_DE_MISC_);
3180 GEN5_IRQ_RESET(GEN8_PCU_);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003181
Paulo Zanoni1c69eb42014-04-01 15:37:23 -03003182 ibx_irq_reset(dev);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003183}
Ben Widawskyabd58f02013-11-02 21:07:09 -07003184
Paulo Zanonid49bdb02014-07-04 11:50:31 -03003185void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv)
3186{
Paulo Zanoni1180e202014-10-07 18:02:52 -03003187 uint32_t extra_ier = GEN8_PIPE_VBLANK | GEN8_PIPE_FIFO_UNDERRUN;
Paulo Zanonid49bdb02014-07-04 11:50:31 -03003188
Daniel Vetter13321782014-09-15 14:55:29 +02003189 spin_lock_irq(&dev_priv->irq_lock);
Paulo Zanonid49bdb02014-07-04 11:50:31 -03003190 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_B, dev_priv->de_irq_mask[PIPE_B],
Paulo Zanoni1180e202014-10-07 18:02:52 -03003191 ~dev_priv->de_irq_mask[PIPE_B] | extra_ier);
Paulo Zanonid49bdb02014-07-04 11:50:31 -03003192 GEN8_IRQ_INIT_NDX(DE_PIPE, PIPE_C, dev_priv->de_irq_mask[PIPE_C],
Paulo Zanoni1180e202014-10-07 18:02:52 -03003193 ~dev_priv->de_irq_mask[PIPE_C] | extra_ier);
Daniel Vetter13321782014-09-15 14:55:29 +02003194 spin_unlock_irq(&dev_priv->irq_lock);
Paulo Zanonid49bdb02014-07-04 11:50:31 -03003195}
3196
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003197static void cherryview_irq_preinstall(struct drm_device *dev)
3198{
3199 struct drm_i915_private *dev_priv = dev->dev_private;
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003200
3201 I915_WRITE(GEN8_MASTER_IRQ, 0);
3202 POSTING_READ(GEN8_MASTER_IRQ);
3203
Daniel Vetterd6e3cca2014-05-22 22:18:22 +02003204 gen8_gt_irq_reset(dev_priv);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003205
3206 GEN5_IRQ_RESET(GEN8_PCU_);
3207
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003208 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK_CHV);
3209
Ville Syrjälä70591a42014-10-30 19:42:58 +02003210 vlv_display_irq_reset(dev_priv);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003211}
3212
Daniel Vetter82a28bc2013-03-27 15:55:01 +01003213static void ibx_hpd_irq_setup(struct drm_device *dev)
Keith Packard7fe0b972011-09-19 13:31:02 -07003214{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003215 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter82a28bc2013-03-27 15:55:01 +01003216 struct intel_encoder *intel_encoder;
Daniel Vetterfee884e2013-07-04 23:35:21 +02003217 u32 hotplug_irqs, hotplug, enabled_irqs = 0;
Keith Packard7fe0b972011-09-19 13:31:02 -07003218
Daniel Vetter82a28bc2013-03-27 15:55:01 +01003219 if (HAS_PCH_IBX(dev)) {
Daniel Vetterfee884e2013-07-04 23:35:21 +02003220 hotplug_irqs = SDE_HOTPLUG_MASK;
Damien Lespiaub2784e12014-08-05 11:29:37 +01003221 for_each_intel_encoder(dev, intel_encoder)
Egbert Eichcd569ae2013-04-16 13:36:57 +02003222 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
Daniel Vetterfee884e2013-07-04 23:35:21 +02003223 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
Daniel Vetter82a28bc2013-03-27 15:55:01 +01003224 } else {
Daniel Vetterfee884e2013-07-04 23:35:21 +02003225 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
Damien Lespiaub2784e12014-08-05 11:29:37 +01003226 for_each_intel_encoder(dev, intel_encoder)
Egbert Eichcd569ae2013-04-16 13:36:57 +02003227 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
Daniel Vetterfee884e2013-07-04 23:35:21 +02003228 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
Daniel Vetter82a28bc2013-03-27 15:55:01 +01003229 }
3230
Daniel Vetterfee884e2013-07-04 23:35:21 +02003231 ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
Daniel Vetter82a28bc2013-03-27 15:55:01 +01003232
3233 /*
3234 * Enable digital hotplug on the PCH, and configure the DP short pulse
3235 * duration to 2ms (which is the minimum in the Display Port spec)
3236 *
3237 * This register is the same on all known PCH chips.
3238 */
Keith Packard7fe0b972011-09-19 13:31:02 -07003239 hotplug = I915_READ(PCH_PORT_HOTPLUG);
3240 hotplug &= ~(PORTD_PULSE_DURATION_MASK|PORTC_PULSE_DURATION_MASK|PORTB_PULSE_DURATION_MASK);
3241 hotplug |= PORTD_HOTPLUG_ENABLE | PORTD_PULSE_DURATION_2ms;
3242 hotplug |= PORTC_HOTPLUG_ENABLE | PORTC_PULSE_DURATION_2ms;
3243 hotplug |= PORTB_HOTPLUG_ENABLE | PORTB_PULSE_DURATION_2ms;
3244 I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
3245}
3246
Paulo Zanonid46da432013-02-08 17:35:15 -02003247static void ibx_irq_postinstall(struct drm_device *dev)
3248{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003249 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter82a28bc2013-03-27 15:55:01 +01003250 u32 mask;
Paulo Zanonid46da432013-02-08 17:35:15 -02003251
Daniel Vetter692a04c2013-05-29 21:43:05 +02003252 if (HAS_PCH_NOP(dev))
3253 return;
3254
Paulo Zanoni105b1222014-04-01 15:37:17 -03003255 if (HAS_PCH_IBX(dev))
Daniel Vetter5c673b62014-03-07 20:34:46 +01003256 mask = SDE_GMBUS | SDE_AUX_MASK | SDE_POISON;
Paulo Zanoni105b1222014-04-01 15:37:17 -03003257 else
Daniel Vetter5c673b62014-03-07 20:34:46 +01003258 mask = SDE_GMBUS_CPT | SDE_AUX_MASK_CPT;
Paulo Zanoni86642812013-04-12 17:57:57 -03003259
Paulo Zanoni337ba012014-04-01 15:37:16 -03003260 GEN5_ASSERT_IIR_IS_ZERO(SDEIIR);
Paulo Zanonid46da432013-02-08 17:35:15 -02003261 I915_WRITE(SDEIMR, ~mask);
Paulo Zanonid46da432013-02-08 17:35:15 -02003262}
3263
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02003264static void gen5_gt_irq_postinstall(struct drm_device *dev)
3265{
3266 struct drm_i915_private *dev_priv = dev->dev_private;
3267 u32 pm_irqs, gt_irqs;
3268
3269 pm_irqs = gt_irqs = 0;
3270
3271 dev_priv->gt_irq_mask = ~0;
Ben Widawsky040d2ba2013-09-19 11:01:40 -07003272 if (HAS_L3_DPF(dev)) {
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02003273 /* L3 parity interrupt is always unmasked. */
Ben Widawsky35a85ac2013-09-19 11:13:41 -07003274 dev_priv->gt_irq_mask = ~GT_PARITY_ERROR(dev);
3275 gt_irqs |= GT_PARITY_ERROR(dev);
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02003276 }
3277
3278 gt_irqs |= GT_RENDER_USER_INTERRUPT;
3279 if (IS_GEN5(dev)) {
3280 gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
3281 ILK_BSD_USER_INTERRUPT;
3282 } else {
3283 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
3284 }
3285
Paulo Zanoni35079892014-04-01 15:37:15 -03003286 GEN5_IRQ_INIT(GT, dev_priv->gt_irq_mask, gt_irqs);
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02003287
3288 if (INTEL_INFO(dev)->gen >= 6) {
Deepak Sa6706b42014-03-15 20:23:22 +05303289 pm_irqs |= dev_priv->pm_rps_events;
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02003290
3291 if (HAS_VEBOX(dev))
3292 pm_irqs |= PM_VEBOX_USER_INTERRUPT;
3293
Paulo Zanoni605cd252013-08-06 18:57:15 -03003294 dev_priv->pm_irq_mask = 0xffffffff;
Paulo Zanoni35079892014-04-01 15:37:15 -03003295 GEN5_IRQ_INIT(GEN6_PM, dev_priv->pm_irq_mask, pm_irqs);
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02003296 }
3297}
3298
Jesse Barnesf71d4af2011-06-28 13:00:41 -07003299static int ironlake_irq_postinstall(struct drm_device *dev)
Zhenyu Wang036a4a72009-06-08 14:40:19 +08003300{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003301 struct drm_i915_private *dev_priv = dev->dev_private;
Paulo Zanoni8e76f8d2013-07-12 20:01:56 -03003302 u32 display_mask, extra_mask;
3303
3304 if (INTEL_INFO(dev)->gen >= 7) {
3305 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
3306 DE_PCH_EVENT_IVB | DE_PLANEC_FLIP_DONE_IVB |
3307 DE_PLANEB_FLIP_DONE_IVB |
Daniel Vetter5c673b62014-03-07 20:34:46 +01003308 DE_PLANEA_FLIP_DONE_IVB | DE_AUX_CHANNEL_A_IVB);
Paulo Zanoni8e76f8d2013-07-12 20:01:56 -03003309 extra_mask = (DE_PIPEC_VBLANK_IVB | DE_PIPEB_VBLANK_IVB |
Daniel Vetter5c673b62014-03-07 20:34:46 +01003310 DE_PIPEA_VBLANK_IVB | DE_ERR_INT_IVB);
Paulo Zanoni8e76f8d2013-07-12 20:01:56 -03003311 } else {
3312 display_mask = (DE_MASTER_IRQ_CONTROL | DE_GSE | DE_PCH_EVENT |
3313 DE_PLANEA_FLIP_DONE | DE_PLANEB_FLIP_DONE |
Daniel Vetter5b3a8562013-10-16 22:55:48 +02003314 DE_AUX_CHANNEL_A |
Daniel Vetter5b3a8562013-10-16 22:55:48 +02003315 DE_PIPEB_CRC_DONE | DE_PIPEA_CRC_DONE |
3316 DE_POISON);
Daniel Vetter5c673b62014-03-07 20:34:46 +01003317 extra_mask = DE_PIPEA_VBLANK | DE_PIPEB_VBLANK | DE_PCU_EVENT |
3318 DE_PIPEB_FIFO_UNDERRUN | DE_PIPEA_FIFO_UNDERRUN;
Paulo Zanoni8e76f8d2013-07-12 20:01:56 -03003319 }
Zhenyu Wang036a4a72009-06-08 14:40:19 +08003320
Chris Wilson1ec14ad2010-12-04 11:30:53 +00003321 dev_priv->irq_mask = ~display_mask;
Zhenyu Wang036a4a72009-06-08 14:40:19 +08003322
Paulo Zanoni0c841212014-04-01 15:37:27 -03003323 I915_WRITE(HWSTAM, 0xeffe);
3324
Paulo Zanoni622364b2014-04-01 15:37:22 -03003325 ibx_irq_pre_postinstall(dev);
3326
Paulo Zanoni35079892014-04-01 15:37:15 -03003327 GEN5_IRQ_INIT(DE, dev_priv->irq_mask, display_mask | extra_mask);
Zhenyu Wang036a4a72009-06-08 14:40:19 +08003328
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02003329 gen5_gt_irq_postinstall(dev);
Zhenyu Wang036a4a72009-06-08 14:40:19 +08003330
Paulo Zanonid46da432013-02-08 17:35:15 -02003331 ibx_irq_postinstall(dev);
Keith Packard7fe0b972011-09-19 13:31:02 -07003332
Jesse Barnesf97108d2010-01-29 11:27:07 -08003333 if (IS_IRONLAKE_M(dev)) {
Daniel Vetter6005ce42013-06-27 13:44:59 +02003334 /* Enable PCU event interrupts
3335 *
3336 * spinlocking not required here for correctness since interrupt
Daniel Vetter4bc9d432013-06-27 13:44:58 +02003337 * setup is guaranteed to run in single-threaded context. But we
3338 * need it to make the assert_spin_locked happy. */
Daniel Vetterd6207432014-09-15 14:55:27 +02003339 spin_lock_irq(&dev_priv->irq_lock);
Jesse Barnesf97108d2010-01-29 11:27:07 -08003340 ironlake_enable_display_irq(dev_priv, DE_PCU_EVENT);
Daniel Vetterd6207432014-09-15 14:55:27 +02003341 spin_unlock_irq(&dev_priv->irq_lock);
Jesse Barnesf97108d2010-01-29 11:27:07 -08003342 }
3343
Zhenyu Wang036a4a72009-06-08 14:40:19 +08003344 return 0;
3345}
3346
Imre Deakf8b79e52014-03-04 19:23:07 +02003347static void valleyview_display_irqs_install(struct drm_i915_private *dev_priv)
3348{
3349 u32 pipestat_mask;
3350 u32 iir_mask;
Ville Syrjälä120dda42014-10-30 19:42:57 +02003351 enum pipe pipe;
Imre Deakf8b79e52014-03-04 19:23:07 +02003352
3353 pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3354 PIPE_FIFO_UNDERRUN_STATUS;
3355
Ville Syrjälä120dda42014-10-30 19:42:57 +02003356 for_each_pipe(dev_priv, pipe)
3357 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
Imre Deakf8b79e52014-03-04 19:23:07 +02003358 POSTING_READ(PIPESTAT(PIPE_A));
3359
3360 pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3361 PIPE_CRC_DONE_INTERRUPT_STATUS;
3362
Ville Syrjälä120dda42014-10-30 19:42:57 +02003363 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3364 for_each_pipe(dev_priv, pipe)
3365 i915_enable_pipestat(dev_priv, pipe, pipestat_mask);
Imre Deakf8b79e52014-03-04 19:23:07 +02003366
3367 iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3368 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3369 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
Ville Syrjälä120dda42014-10-30 19:42:57 +02003370 if (IS_CHERRYVIEW(dev_priv))
3371 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
Imre Deakf8b79e52014-03-04 19:23:07 +02003372 dev_priv->irq_mask &= ~iir_mask;
3373
3374 I915_WRITE(VLV_IIR, iir_mask);
3375 I915_WRITE(VLV_IIR, iir_mask);
Imre Deakf8b79e52014-03-04 19:23:07 +02003376 I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
Ville Syrjälä76e41862014-10-30 19:42:54 +02003377 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3378 POSTING_READ(VLV_IMR);
Imre Deakf8b79e52014-03-04 19:23:07 +02003379}
3380
3381static void valleyview_display_irqs_uninstall(struct drm_i915_private *dev_priv)
3382{
3383 u32 pipestat_mask;
3384 u32 iir_mask;
Ville Syrjälä120dda42014-10-30 19:42:57 +02003385 enum pipe pipe;
Imre Deakf8b79e52014-03-04 19:23:07 +02003386
3387 iir_mask = I915_DISPLAY_PORT_INTERRUPT |
3388 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
Imre Deak6c7fba02014-03-10 19:44:48 +02003389 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT;
Ville Syrjälä120dda42014-10-30 19:42:57 +02003390 if (IS_CHERRYVIEW(dev_priv))
3391 iir_mask |= I915_DISPLAY_PIPE_C_EVENT_INTERRUPT;
Imre Deakf8b79e52014-03-04 19:23:07 +02003392
3393 dev_priv->irq_mask |= iir_mask;
Imre Deakf8b79e52014-03-04 19:23:07 +02003394 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
Ville Syrjälä76e41862014-10-30 19:42:54 +02003395 I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
Imre Deakf8b79e52014-03-04 19:23:07 +02003396 I915_WRITE(VLV_IIR, iir_mask);
3397 I915_WRITE(VLV_IIR, iir_mask);
3398 POSTING_READ(VLV_IIR);
3399
3400 pipestat_mask = PLANE_FLIP_DONE_INT_STATUS_VLV |
3401 PIPE_CRC_DONE_INTERRUPT_STATUS;
3402
Ville Syrjälä120dda42014-10-30 19:42:57 +02003403 i915_disable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
3404 for_each_pipe(dev_priv, pipe)
3405 i915_disable_pipestat(dev_priv, pipe, pipestat_mask);
Imre Deakf8b79e52014-03-04 19:23:07 +02003406
3407 pipestat_mask = PIPESTAT_INT_STATUS_MASK |
3408 PIPE_FIFO_UNDERRUN_STATUS;
Ville Syrjälä120dda42014-10-30 19:42:57 +02003409
3410 for_each_pipe(dev_priv, pipe)
3411 I915_WRITE(PIPESTAT(pipe), pipestat_mask);
Imre Deakf8b79e52014-03-04 19:23:07 +02003412 POSTING_READ(PIPESTAT(PIPE_A));
3413}
3414
3415void valleyview_enable_display_irqs(struct drm_i915_private *dev_priv)
3416{
3417 assert_spin_locked(&dev_priv->irq_lock);
3418
3419 if (dev_priv->display_irqs_enabled)
3420 return;
3421
3422 dev_priv->display_irqs_enabled = true;
3423
Imre Deak950eaba2014-09-08 15:21:09 +03003424 if (intel_irqs_enabled(dev_priv))
Imre Deakf8b79e52014-03-04 19:23:07 +02003425 valleyview_display_irqs_install(dev_priv);
3426}
3427
3428void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
3429{
3430 assert_spin_locked(&dev_priv->irq_lock);
3431
3432 if (!dev_priv->display_irqs_enabled)
3433 return;
3434
3435 dev_priv->display_irqs_enabled = false;
3436
Imre Deak950eaba2014-09-08 15:21:09 +03003437 if (intel_irqs_enabled(dev_priv))
Imre Deakf8b79e52014-03-04 19:23:07 +02003438 valleyview_display_irqs_uninstall(dev_priv);
3439}
3440
Ville Syrjälä0e6c9a92014-10-30 19:43:00 +02003441static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003442{
Imre Deakf8b79e52014-03-04 19:23:07 +02003443 dev_priv->irq_mask = ~0;
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003444
Daniel Vetter20afbda2012-12-11 14:05:07 +01003445 I915_WRITE(PORT_HOTPLUG_EN, 0);
3446 POSTING_READ(PORT_HOTPLUG_EN);
3447
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003448 I915_WRITE(VLV_IIR, 0xffffffff);
Ville Syrjälä76e41862014-10-30 19:42:54 +02003449 I915_WRITE(VLV_IIR, 0xffffffff);
3450 I915_WRITE(VLV_IER, ~dev_priv->irq_mask);
3451 I915_WRITE(VLV_IMR, dev_priv->irq_mask);
3452 POSTING_READ(VLV_IMR);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003453
Daniel Vetterb79480b2013-06-27 17:52:10 +02003454 /* Interrupt setup is already guaranteed to be single-threaded, this is
3455 * just to make the assert_spin_locked check happy. */
Daniel Vetterd6207432014-09-15 14:55:27 +02003456 spin_lock_irq(&dev_priv->irq_lock);
Imre Deakf8b79e52014-03-04 19:23:07 +02003457 if (dev_priv->display_irqs_enabled)
3458 valleyview_display_irqs_install(dev_priv);
Daniel Vetterd6207432014-09-15 14:55:27 +02003459 spin_unlock_irq(&dev_priv->irq_lock);
Ville Syrjälä0e6c9a92014-10-30 19:43:00 +02003460}
3461
3462static int valleyview_irq_postinstall(struct drm_device *dev)
3463{
3464 struct drm_i915_private *dev_priv = dev->dev_private;
3465
3466 vlv_display_irq_postinstall(dev_priv);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003467
Daniel Vetter0a9a8c92013-07-12 22:43:26 +02003468 gen5_gt_irq_postinstall(dev);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003469
3470 /* ack & enable invalid PTE error interrupts */
3471#if 0 /* FIXME: add support to irq handler for checking these bits */
3472 I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
3473 I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
3474#endif
3475
3476 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
Daniel Vetter20afbda2012-12-11 14:05:07 +01003477
3478 return 0;
3479}
3480
Ben Widawskyabd58f02013-11-02 21:07:09 -07003481static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
3482{
Ben Widawskyabd58f02013-11-02 21:07:09 -07003483 /* These are interrupts we'll toggle with the ring mask register */
3484 uint32_t gt_interrupts[] = {
3485 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
Oscar Mateo73d477f2014-07-24 17:04:31 +01003486 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
Ben Widawskyabd58f02013-11-02 21:07:09 -07003487 GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
Oscar Mateo73d477f2014-07-24 17:04:31 +01003488 GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
3489 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
Ben Widawskyabd58f02013-11-02 21:07:09 -07003490 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
Oscar Mateo73d477f2014-07-24 17:04:31 +01003491 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
3492 GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
3493 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
Ben Widawskyabd58f02013-11-02 21:07:09 -07003494 0,
Oscar Mateo73d477f2014-07-24 17:04:31 +01003495 GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
3496 GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
Ben Widawskyabd58f02013-11-02 21:07:09 -07003497 };
3498
Ben Widawsky09610212014-05-15 20:58:08 +03003499 dev_priv->pm_irq_mask = 0xffffffff;
Deepak S9a2d2d82014-08-22 08:32:40 +05303500 GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
3501 GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
3502 GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, dev_priv->pm_rps_events);
3503 GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003504}
3505
3506static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
3507{
Damien Lespiau770de832014-03-20 20:45:01 +00003508 uint32_t de_pipe_masked = GEN8_PIPE_CDCLK_CRC_DONE;
3509 uint32_t de_pipe_enables;
Ben Widawskyabd58f02013-11-02 21:07:09 -07003510 int pipe;
Jesse Barnes88e04702014-11-13 17:51:48 +00003511 u32 aux_en = GEN8_AUX_CHANNEL_A;
Damien Lespiau770de832014-03-20 20:45:01 +00003512
Jesse Barnes88e04702014-11-13 17:51:48 +00003513 if (IS_GEN9(dev_priv)) {
Damien Lespiau770de832014-03-20 20:45:01 +00003514 de_pipe_masked |= GEN9_PIPE_PLANE1_FLIP_DONE |
3515 GEN9_DE_PIPE_IRQ_FAULT_ERRORS;
Jesse Barnes88e04702014-11-13 17:51:48 +00003516 aux_en |= GEN9_AUX_CHANNEL_B | GEN9_AUX_CHANNEL_C |
3517 GEN9_AUX_CHANNEL_D;
3518 } else
Damien Lespiau770de832014-03-20 20:45:01 +00003519 de_pipe_masked |= GEN8_PIPE_PRIMARY_FLIP_DONE |
3520 GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
3521
3522 de_pipe_enables = de_pipe_masked | GEN8_PIPE_VBLANK |
3523 GEN8_PIPE_FIFO_UNDERRUN;
3524
Daniel Vetter13b3a0a2013-11-07 15:31:52 +01003525 dev_priv->de_irq_mask[PIPE_A] = ~de_pipe_masked;
3526 dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
3527 dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
Ben Widawskyabd58f02013-11-02 21:07:09 -07003528
Damien Lespiau055e3932014-08-18 13:49:10 +01003529 for_each_pipe(dev_priv, pipe)
Daniel Vetterf458ebb2014-09-30 10:56:39 +02003530 if (intel_display_power_is_enabled(dev_priv,
Paulo Zanoni813bde42014-07-04 11:50:29 -03003531 POWER_DOMAIN_PIPE(pipe)))
3532 GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
3533 dev_priv->de_irq_mask[pipe],
3534 de_pipe_enables);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003535
Jesse Barnes88e04702014-11-13 17:51:48 +00003536 GEN5_IRQ_INIT(GEN8_DE_PORT_, ~aux_en, aux_en);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003537}
3538
3539static int gen8_irq_postinstall(struct drm_device *dev)
3540{
3541 struct drm_i915_private *dev_priv = dev->dev_private;
3542
Paulo Zanoni622364b2014-04-01 15:37:22 -03003543 ibx_irq_pre_postinstall(dev);
3544
Ben Widawskyabd58f02013-11-02 21:07:09 -07003545 gen8_gt_irq_postinstall(dev_priv);
3546 gen8_de_irq_postinstall(dev_priv);
3547
3548 ibx_irq_postinstall(dev);
3549
3550 I915_WRITE(GEN8_MASTER_IRQ, DE_MASTER_IRQ_CONTROL);
3551 POSTING_READ(GEN8_MASTER_IRQ);
3552
3553 return 0;
3554}
3555
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003556static int cherryview_irq_postinstall(struct drm_device *dev)
3557{
3558 struct drm_i915_private *dev_priv = dev->dev_private;
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003559
Ville Syrjäläc2b66792014-10-30 19:43:02 +02003560 vlv_display_irq_postinstall(dev_priv);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003561
3562 gen8_gt_irq_postinstall(dev_priv);
3563
3564 I915_WRITE(GEN8_MASTER_IRQ, MASTER_INTERRUPT_ENABLE);
3565 POSTING_READ(GEN8_MASTER_IRQ);
3566
3567 return 0;
3568}
3569
Ben Widawskyabd58f02013-11-02 21:07:09 -07003570static void gen8_irq_uninstall(struct drm_device *dev)
3571{
3572 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawskyabd58f02013-11-02 21:07:09 -07003573
3574 if (!dev_priv)
3575 return;
3576
Paulo Zanoni823f6b32014-04-01 15:37:26 -03003577 gen8_irq_reset(dev);
Ben Widawskyabd58f02013-11-02 21:07:09 -07003578}
3579
Ville Syrjälä8ea0be42014-10-30 19:42:59 +02003580static void vlv_display_irq_uninstall(struct drm_i915_private *dev_priv)
3581{
3582 /* Interrupt setup is already guaranteed to be single-threaded, this is
3583 * just to make the assert_spin_locked check happy. */
3584 spin_lock_irq(&dev_priv->irq_lock);
3585 if (dev_priv->display_irqs_enabled)
3586 valleyview_display_irqs_uninstall(dev_priv);
3587 spin_unlock_irq(&dev_priv->irq_lock);
3588
3589 vlv_display_irq_reset(dev_priv);
3590
3591 dev_priv->irq_mask = 0;
3592}
3593
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003594static void valleyview_irq_uninstall(struct drm_device *dev)
3595{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003596 struct drm_i915_private *dev_priv = dev->dev_private;
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003597
3598 if (!dev_priv)
3599 return;
3600
Imre Deak843d0e72014-04-14 20:24:23 +03003601 I915_WRITE(VLV_MASTER_IER, 0);
3602
Ville Syrjälä893fce82014-10-30 19:42:56 +02003603 gen5_gt_irq_reset(dev);
3604
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003605 I915_WRITE(HWSTAM, 0xffffffff);
Imre Deakf8b79e52014-03-04 19:23:07 +02003606
Ville Syrjälä8ea0be42014-10-30 19:42:59 +02003607 vlv_display_irq_uninstall(dev_priv);
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07003608}
3609
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003610static void cherryview_irq_uninstall(struct drm_device *dev)
3611{
3612 struct drm_i915_private *dev_priv = dev->dev_private;
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003613
3614 if (!dev_priv)
3615 return;
3616
3617 I915_WRITE(GEN8_MASTER_IRQ, 0);
3618 POSTING_READ(GEN8_MASTER_IRQ);
3619
Ville Syrjäläa2c30fb2014-10-30 19:42:52 +02003620 gen8_gt_irq_reset(dev_priv);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003621
Ville Syrjäläa2c30fb2014-10-30 19:42:52 +02003622 GEN5_IRQ_RESET(GEN8_PCU_);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003623
Ville Syrjäläc2b66792014-10-30 19:43:02 +02003624 vlv_display_irq_uninstall(dev_priv);
Ville Syrjälä43f328d2014-04-09 20:40:52 +03003625}
3626
Jesse Barnesf71d4af2011-06-28 13:00:41 -07003627static void ironlake_irq_uninstall(struct drm_device *dev)
Zhenyu Wang036a4a72009-06-08 14:40:19 +08003628{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003629 struct drm_i915_private *dev_priv = dev->dev_private;
Jesse Barnes46979952011-04-07 13:53:55 -07003630
3631 if (!dev_priv)
3632 return;
3633
Paulo Zanonibe30b292014-04-01 15:37:25 -03003634 ironlake_irq_reset(dev);
Zhenyu Wang036a4a72009-06-08 14:40:19 +08003635}
3636
Chris Wilsonc2798b12012-04-22 21:13:57 +01003637static void i8xx_irq_preinstall(struct drm_device * dev)
3638{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003639 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilsonc2798b12012-04-22 21:13:57 +01003640 int pipe;
3641
Damien Lespiau055e3932014-08-18 13:49:10 +01003642 for_each_pipe(dev_priv, pipe)
Chris Wilsonc2798b12012-04-22 21:13:57 +01003643 I915_WRITE(PIPESTAT(pipe), 0);
3644 I915_WRITE16(IMR, 0xffff);
3645 I915_WRITE16(IER, 0x0);
3646 POSTING_READ16(IER);
3647}
3648
3649static int i8xx_irq_postinstall(struct drm_device *dev)
3650{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003651 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilsonc2798b12012-04-22 21:13:57 +01003652
Chris Wilsonc2798b12012-04-22 21:13:57 +01003653 I915_WRITE16(EMR,
3654 ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3655
3656 /* Unmask the interrupts that we always want on. */
3657 dev_priv->irq_mask =
3658 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3659 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3660 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3661 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
3662 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3663 I915_WRITE16(IMR, dev_priv->irq_mask);
3664
3665 I915_WRITE16(IER,
3666 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3667 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3668 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
3669 I915_USER_INTERRUPT);
3670 POSTING_READ16(IER);
3671
Daniel Vetter379ef822013-10-16 22:55:56 +02003672 /* Interrupt setup is already guaranteed to be single-threaded, this is
3673 * just to make the assert_spin_locked check happy. */
Daniel Vetterd6207432014-09-15 14:55:27 +02003674 spin_lock_irq(&dev_priv->irq_lock);
Imre Deak755e9012014-02-10 18:42:47 +02003675 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3676 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
Daniel Vetterd6207432014-09-15 14:55:27 +02003677 spin_unlock_irq(&dev_priv->irq_lock);
Daniel Vetter379ef822013-10-16 22:55:56 +02003678
Chris Wilsonc2798b12012-04-22 21:13:57 +01003679 return 0;
3680}
3681
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003682/*
3683 * Returns true when a page flip has completed.
3684 */
3685static bool i8xx_handle_vblank(struct drm_device *dev,
Ville Syrjälä1f1c2e22013-11-28 17:30:01 +02003686 int plane, int pipe, u32 iir)
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003687{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003688 struct drm_i915_private *dev_priv = dev->dev_private;
Ville Syrjälä1f1c2e22013-11-28 17:30:01 +02003689 u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003690
Ville Syrjälä8d7849d2014-04-29 13:35:46 +03003691 if (!intel_pipe_handle_vblank(dev, pipe))
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003692 return false;
3693
3694 if ((iir & flip_pending) == 0)
Chris Wilsond6bbafa2014-09-05 07:13:24 +01003695 goto check_page_flip;
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003696
Ville Syrjälä1f1c2e22013-11-28 17:30:01 +02003697 intel_prepare_page_flip(dev, plane);
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003698
3699 /* We detect FlipDone by looking for the change in PendingFlip from '1'
3700 * to '0' on the following vblank, i.e. IIR has the Pendingflip
3701 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3702 * the flip is completed (no longer pending). Since this doesn't raise
3703 * an interrupt per se, we watch for the change at vblank.
3704 */
3705 if (I915_READ16(ISR) & flip_pending)
Chris Wilsond6bbafa2014-09-05 07:13:24 +01003706 goto check_page_flip;
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003707
3708 intel_finish_page_flip(dev, pipe);
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003709 return true;
Chris Wilsond6bbafa2014-09-05 07:13:24 +01003710
3711check_page_flip:
3712 intel_check_page_flip(dev, pipe);
3713 return false;
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003714}
3715
Daniel Vetterff1f5252012-10-02 15:10:55 +02003716static irqreturn_t i8xx_irq_handler(int irq, void *arg)
Chris Wilsonc2798b12012-04-22 21:13:57 +01003717{
Daniel Vetter45a83f82014-05-12 19:17:55 +02003718 struct drm_device *dev = arg;
Jani Nikula2d1013d2014-03-31 14:27:17 +03003719 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilsonc2798b12012-04-22 21:13:57 +01003720 u16 iir, new_iir;
3721 u32 pipe_stats[2];
Chris Wilsonc2798b12012-04-22 21:13:57 +01003722 int pipe;
3723 u16 flip_mask =
3724 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3725 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
3726
Chris Wilsonc2798b12012-04-22 21:13:57 +01003727 iir = I915_READ16(IIR);
3728 if (iir == 0)
3729 return IRQ_NONE;
3730
3731 while (iir & ~flip_mask) {
3732 /* Can't rely on pipestat interrupt bit in iir as it might
3733 * have been cleared after the pipestat interrupt was received.
3734 * It doesn't set the bit in iir again, but it still produces
3735 * interrupts (for non-MSI).
3736 */
Daniel Vetter222c7f52014-09-15 14:55:28 +02003737 spin_lock(&dev_priv->irq_lock);
Chris Wilsonc2798b12012-04-22 21:13:57 +01003738 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
Mika Kuoppala58174462014-02-25 17:11:26 +02003739 i915_handle_error(dev, false,
3740 "Command parser error, iir 0x%08x",
3741 iir);
Chris Wilsonc2798b12012-04-22 21:13:57 +01003742
Damien Lespiau055e3932014-08-18 13:49:10 +01003743 for_each_pipe(dev_priv, pipe) {
Chris Wilsonc2798b12012-04-22 21:13:57 +01003744 int reg = PIPESTAT(pipe);
3745 pipe_stats[pipe] = I915_READ(reg);
3746
3747 /*
3748 * Clear the PIPE*STAT regs before the IIR
3749 */
Ville Syrjälä2d9d2b02014-01-17 11:44:31 +02003750 if (pipe_stats[pipe] & 0x8000ffff)
Chris Wilsonc2798b12012-04-22 21:13:57 +01003751 I915_WRITE(reg, pipe_stats[pipe]);
Chris Wilsonc2798b12012-04-22 21:13:57 +01003752 }
Daniel Vetter222c7f52014-09-15 14:55:28 +02003753 spin_unlock(&dev_priv->irq_lock);
Chris Wilsonc2798b12012-04-22 21:13:57 +01003754
3755 I915_WRITE16(IIR, iir & ~flip_mask);
3756 new_iir = I915_READ16(IIR); /* Flush posted writes */
3757
Daniel Vetterd05c6172012-04-26 23:28:09 +02003758 i915_update_dri1_breadcrumb(dev);
Chris Wilsonc2798b12012-04-22 21:13:57 +01003759
3760 if (iir & I915_USER_INTERRUPT)
3761 notify_ring(dev, &dev_priv->ring[RCS]);
3762
Damien Lespiau055e3932014-08-18 13:49:10 +01003763 for_each_pipe(dev_priv, pipe) {
Ville Syrjälä1f1c2e22013-11-28 17:30:01 +02003764 int plane = pipe;
Daniel Vetter3a77c4c2014-01-10 08:50:12 +01003765 if (HAS_FBC(dev))
Ville Syrjälä1f1c2e22013-11-28 17:30:01 +02003766 plane = !plane;
3767
Daniel Vetter4356d582013-10-16 22:55:55 +02003768 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
Ville Syrjälä1f1c2e22013-11-28 17:30:01 +02003769 i8xx_handle_vblank(dev, plane, pipe, iir))
3770 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
Chris Wilsonc2798b12012-04-22 21:13:57 +01003771
Daniel Vetter4356d582013-10-16 22:55:55 +02003772 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
Daniel Vetter277de952013-10-18 16:37:07 +02003773 i9xx_pipe_crc_irq_handler(dev, pipe);
Ville Syrjälä2d9d2b02014-01-17 11:44:31 +02003774
Daniel Vetter1f7247c2014-09-30 10:56:48 +02003775 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3776 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3777 pipe);
Daniel Vetter4356d582013-10-16 22:55:55 +02003778 }
Chris Wilsonc2798b12012-04-22 21:13:57 +01003779
3780 iir = new_iir;
3781 }
3782
3783 return IRQ_HANDLED;
3784}
3785
3786static void i8xx_irq_uninstall(struct drm_device * dev)
3787{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003788 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilsonc2798b12012-04-22 21:13:57 +01003789 int pipe;
3790
Damien Lespiau055e3932014-08-18 13:49:10 +01003791 for_each_pipe(dev_priv, pipe) {
Chris Wilsonc2798b12012-04-22 21:13:57 +01003792 /* Clear enable bits; then clear status bits */
3793 I915_WRITE(PIPESTAT(pipe), 0);
3794 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
3795 }
3796 I915_WRITE16(IMR, 0xffff);
3797 I915_WRITE16(IER, 0x0);
3798 I915_WRITE16(IIR, I915_READ16(IIR));
3799}
3800
Chris Wilsona266c7d2012-04-24 22:59:44 +01003801static void i915_irq_preinstall(struct drm_device * dev)
3802{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003803 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilsona266c7d2012-04-24 22:59:44 +01003804 int pipe;
3805
Chris Wilsona266c7d2012-04-24 22:59:44 +01003806 if (I915_HAS_HOTPLUG(dev)) {
3807 I915_WRITE(PORT_HOTPLUG_EN, 0);
3808 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
3809 }
3810
Chris Wilson00d98eb2012-04-24 22:59:48 +01003811 I915_WRITE16(HWSTAM, 0xeffe);
Damien Lespiau055e3932014-08-18 13:49:10 +01003812 for_each_pipe(dev_priv, pipe)
Chris Wilsona266c7d2012-04-24 22:59:44 +01003813 I915_WRITE(PIPESTAT(pipe), 0);
3814 I915_WRITE(IMR, 0xffffffff);
3815 I915_WRITE(IER, 0x0);
3816 POSTING_READ(IER);
3817}
3818
3819static int i915_irq_postinstall(struct drm_device *dev)
3820{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003821 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson38bde182012-04-24 22:59:50 +01003822 u32 enable_mask;
Chris Wilsona266c7d2012-04-24 22:59:44 +01003823
Chris Wilson38bde182012-04-24 22:59:50 +01003824 I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
3825
3826 /* Unmask the interrupts that we always want on. */
3827 dev_priv->irq_mask =
3828 ~(I915_ASLE_INTERRUPT |
3829 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3830 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3831 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3832 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
3833 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
3834
3835 enable_mask =
3836 I915_ASLE_INTERRUPT |
3837 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
3838 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
3839 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
3840 I915_USER_INTERRUPT;
3841
Chris Wilsona266c7d2012-04-24 22:59:44 +01003842 if (I915_HAS_HOTPLUG(dev)) {
Daniel Vetter20afbda2012-12-11 14:05:07 +01003843 I915_WRITE(PORT_HOTPLUG_EN, 0);
3844 POSTING_READ(PORT_HOTPLUG_EN);
3845
Chris Wilsona266c7d2012-04-24 22:59:44 +01003846 /* Enable in IER... */
3847 enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
3848 /* and unmask in IMR */
3849 dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
3850 }
3851
Chris Wilsona266c7d2012-04-24 22:59:44 +01003852 I915_WRITE(IMR, dev_priv->irq_mask);
3853 I915_WRITE(IER, enable_mask);
3854 POSTING_READ(IER);
3855
Jani Nikulaf49e38d2013-04-29 13:02:54 +03003856 i915_enable_asle_pipestat(dev);
Daniel Vetter20afbda2012-12-11 14:05:07 +01003857
Daniel Vetter379ef822013-10-16 22:55:56 +02003858 /* Interrupt setup is already guaranteed to be single-threaded, this is
3859 * just to make the assert_spin_locked check happy. */
Daniel Vetterd6207432014-09-15 14:55:27 +02003860 spin_lock_irq(&dev_priv->irq_lock);
Imre Deak755e9012014-02-10 18:42:47 +02003861 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
3862 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
Daniel Vetterd6207432014-09-15 14:55:27 +02003863 spin_unlock_irq(&dev_priv->irq_lock);
Daniel Vetter379ef822013-10-16 22:55:56 +02003864
Daniel Vetter20afbda2012-12-11 14:05:07 +01003865 return 0;
3866}
3867
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003868/*
3869 * Returns true when a page flip has completed.
3870 */
3871static bool i915_handle_vblank(struct drm_device *dev,
3872 int plane, int pipe, u32 iir)
3873{
Jani Nikula2d1013d2014-03-31 14:27:17 +03003874 struct drm_i915_private *dev_priv = dev->dev_private;
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003875 u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
3876
Ville Syrjälä8d7849d2014-04-29 13:35:46 +03003877 if (!intel_pipe_handle_vblank(dev, pipe))
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003878 return false;
3879
3880 if ((iir & flip_pending) == 0)
Chris Wilsond6bbafa2014-09-05 07:13:24 +01003881 goto check_page_flip;
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003882
3883 intel_prepare_page_flip(dev, plane);
3884
3885 /* We detect FlipDone by looking for the change in PendingFlip from '1'
3886 * to '0' on the following vblank, i.e. IIR has the Pendingflip
3887 * asserted following the MI_DISPLAY_FLIP, but ISR is deasserted, hence
3888 * the flip is completed (no longer pending). Since this doesn't raise
3889 * an interrupt per se, we watch for the change at vblank.
3890 */
3891 if (I915_READ(ISR) & flip_pending)
Chris Wilsond6bbafa2014-09-05 07:13:24 +01003892 goto check_page_flip;
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003893
3894 intel_finish_page_flip(dev, pipe);
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003895 return true;
Chris Wilsond6bbafa2014-09-05 07:13:24 +01003896
3897check_page_flip:
3898 intel_check_page_flip(dev, pipe);
3899 return false;
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003900}
3901
Daniel Vetterff1f5252012-10-02 15:10:55 +02003902static irqreturn_t i915_irq_handler(int irq, void *arg)
Chris Wilsona266c7d2012-04-24 22:59:44 +01003903{
Daniel Vetter45a83f82014-05-12 19:17:55 +02003904 struct drm_device *dev = arg;
Jani Nikula2d1013d2014-03-31 14:27:17 +03003905 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson8291ee92012-04-24 22:59:47 +01003906 u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
Chris Wilson38bde182012-04-24 22:59:50 +01003907 u32 flip_mask =
3908 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
3909 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
Chris Wilson38bde182012-04-24 22:59:50 +01003910 int pipe, ret = IRQ_NONE;
Chris Wilsona266c7d2012-04-24 22:59:44 +01003911
Chris Wilsona266c7d2012-04-24 22:59:44 +01003912 iir = I915_READ(IIR);
Chris Wilson38bde182012-04-24 22:59:50 +01003913 do {
3914 bool irq_received = (iir & ~flip_mask) != 0;
Chris Wilson8291ee92012-04-24 22:59:47 +01003915 bool blc_event = false;
Chris Wilsona266c7d2012-04-24 22:59:44 +01003916
3917 /* Can't rely on pipestat interrupt bit in iir as it might
3918 * have been cleared after the pipestat interrupt was received.
3919 * It doesn't set the bit in iir again, but it still produces
3920 * interrupts (for non-MSI).
3921 */
Daniel Vetter222c7f52014-09-15 14:55:28 +02003922 spin_lock(&dev_priv->irq_lock);
Chris Wilsona266c7d2012-04-24 22:59:44 +01003923 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
Mika Kuoppala58174462014-02-25 17:11:26 +02003924 i915_handle_error(dev, false,
3925 "Command parser error, iir 0x%08x",
3926 iir);
Chris Wilsona266c7d2012-04-24 22:59:44 +01003927
Damien Lespiau055e3932014-08-18 13:49:10 +01003928 for_each_pipe(dev_priv, pipe) {
Chris Wilsona266c7d2012-04-24 22:59:44 +01003929 int reg = PIPESTAT(pipe);
3930 pipe_stats[pipe] = I915_READ(reg);
3931
Chris Wilson38bde182012-04-24 22:59:50 +01003932 /* Clear the PIPE*STAT regs before the IIR */
Chris Wilsona266c7d2012-04-24 22:59:44 +01003933 if (pipe_stats[pipe] & 0x8000ffff) {
Chris Wilsona266c7d2012-04-24 22:59:44 +01003934 I915_WRITE(reg, pipe_stats[pipe]);
Chris Wilson38bde182012-04-24 22:59:50 +01003935 irq_received = true;
Chris Wilsona266c7d2012-04-24 22:59:44 +01003936 }
3937 }
Daniel Vetter222c7f52014-09-15 14:55:28 +02003938 spin_unlock(&dev_priv->irq_lock);
Chris Wilsona266c7d2012-04-24 22:59:44 +01003939
3940 if (!irq_received)
3941 break;
3942
Chris Wilsona266c7d2012-04-24 22:59:44 +01003943 /* Consume port. Then clear IIR or we'll miss events */
Ville Syrjälä16c6c562014-04-01 10:54:36 +03003944 if (I915_HAS_HOTPLUG(dev) &&
3945 iir & I915_DISPLAY_PORT_INTERRUPT)
3946 i9xx_hpd_irq_handler(dev);
Chris Wilsona266c7d2012-04-24 22:59:44 +01003947
Chris Wilson38bde182012-04-24 22:59:50 +01003948 I915_WRITE(IIR, iir & ~flip_mask);
Chris Wilsona266c7d2012-04-24 22:59:44 +01003949 new_iir = I915_READ(IIR); /* Flush posted writes */
3950
Chris Wilsona266c7d2012-04-24 22:59:44 +01003951 if (iir & I915_USER_INTERRUPT)
3952 notify_ring(dev, &dev_priv->ring[RCS]);
Chris Wilsona266c7d2012-04-24 22:59:44 +01003953
Damien Lespiau055e3932014-08-18 13:49:10 +01003954 for_each_pipe(dev_priv, pipe) {
Chris Wilson38bde182012-04-24 22:59:50 +01003955 int plane = pipe;
Daniel Vetter3a77c4c2014-01-10 08:50:12 +01003956 if (HAS_FBC(dev))
Chris Wilson38bde182012-04-24 22:59:50 +01003957 plane = !plane;
Ville Syrjälä5e2032d2013-02-19 15:16:38 +02003958
Ville Syrjälä90a72f82013-02-19 23:16:44 +02003959 if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
3960 i915_handle_vblank(dev, plane, pipe, iir))
3961 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
Chris Wilsona266c7d2012-04-24 22:59:44 +01003962
3963 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
3964 blc_event = true;
Daniel Vetter4356d582013-10-16 22:55:55 +02003965
3966 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
Daniel Vetter277de952013-10-18 16:37:07 +02003967 i9xx_pipe_crc_irq_handler(dev, pipe);
Ville Syrjälä2d9d2b02014-01-17 11:44:31 +02003968
Daniel Vetter1f7247c2014-09-30 10:56:48 +02003969 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
3970 intel_cpu_fifo_underrun_irq_handler(dev_priv,
3971 pipe);
Chris Wilsona266c7d2012-04-24 22:59:44 +01003972 }
3973
Chris Wilsona266c7d2012-04-24 22:59:44 +01003974 if (blc_event || (iir & I915_ASLE_INTERRUPT))
3975 intel_opregion_asle_intr(dev);
3976
3977 /* With MSI, interrupts are only generated when iir
3978 * transitions from zero to nonzero. If another bit got
3979 * set while we were handling the existing iir bits, then
3980 * we would never get another interrupt.
3981 *
3982 * This is fine on non-MSI as well, as if we hit this path
3983 * we avoid exiting the interrupt handler only to generate
3984 * another one.
3985 *
3986 * Note that for MSI this could cause a stray interrupt report
3987 * if an interrupt landed in the time between writing IIR and
3988 * the posting read. This should be rare enough to never
3989 * trigger the 99% of 100,000 interrupts test for disabling
3990 * stray interrupts.
3991 */
Chris Wilson38bde182012-04-24 22:59:50 +01003992 ret = IRQ_HANDLED;
Chris Wilsona266c7d2012-04-24 22:59:44 +01003993 iir = new_iir;
Chris Wilson38bde182012-04-24 22:59:50 +01003994 } while (iir & ~flip_mask);
Chris Wilsona266c7d2012-04-24 22:59:44 +01003995
Daniel Vetterd05c6172012-04-26 23:28:09 +02003996 i915_update_dri1_breadcrumb(dev);
Chris Wilson8291ee92012-04-24 22:59:47 +01003997
Chris Wilsona266c7d2012-04-24 22:59:44 +01003998 return ret;
3999}
4000
4001static void i915_irq_uninstall(struct drm_device * dev)
4002{
Jani Nikula2d1013d2014-03-31 14:27:17 +03004003 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004004 int pipe;
4005
Chris Wilsona266c7d2012-04-24 22:59:44 +01004006 if (I915_HAS_HOTPLUG(dev)) {
4007 I915_WRITE(PORT_HOTPLUG_EN, 0);
4008 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
4009 }
4010
Chris Wilson00d98eb2012-04-24 22:59:48 +01004011 I915_WRITE16(HWSTAM, 0xffff);
Damien Lespiau055e3932014-08-18 13:49:10 +01004012 for_each_pipe(dev_priv, pipe) {
Chris Wilson55b39752012-04-24 22:59:49 +01004013 /* Clear enable bits; then clear status bits */
Chris Wilsona266c7d2012-04-24 22:59:44 +01004014 I915_WRITE(PIPESTAT(pipe), 0);
Chris Wilson55b39752012-04-24 22:59:49 +01004015 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
4016 }
Chris Wilsona266c7d2012-04-24 22:59:44 +01004017 I915_WRITE(IMR, 0xffffffff);
4018 I915_WRITE(IER, 0x0);
4019
Chris Wilsona266c7d2012-04-24 22:59:44 +01004020 I915_WRITE(IIR, I915_READ(IIR));
4021}
4022
4023static void i965_irq_preinstall(struct drm_device * dev)
4024{
Jani Nikula2d1013d2014-03-31 14:27:17 +03004025 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004026 int pipe;
4027
Chris Wilsonadca4732012-05-11 18:01:31 +01004028 I915_WRITE(PORT_HOTPLUG_EN, 0);
4029 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
Chris Wilsona266c7d2012-04-24 22:59:44 +01004030
4031 I915_WRITE(HWSTAM, 0xeffe);
Damien Lespiau055e3932014-08-18 13:49:10 +01004032 for_each_pipe(dev_priv, pipe)
Chris Wilsona266c7d2012-04-24 22:59:44 +01004033 I915_WRITE(PIPESTAT(pipe), 0);
4034 I915_WRITE(IMR, 0xffffffff);
4035 I915_WRITE(IER, 0x0);
4036 POSTING_READ(IER);
4037}
4038
4039static int i965_irq_postinstall(struct drm_device *dev)
4040{
Jani Nikula2d1013d2014-03-31 14:27:17 +03004041 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilsonbbba0a92012-04-24 22:59:51 +01004042 u32 enable_mask;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004043 u32 error_mask;
4044
Chris Wilsona266c7d2012-04-24 22:59:44 +01004045 /* Unmask the interrupts that we always want on. */
Chris Wilsonbbba0a92012-04-24 22:59:51 +01004046 dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
Chris Wilsonadca4732012-05-11 18:01:31 +01004047 I915_DISPLAY_PORT_INTERRUPT |
Chris Wilsonbbba0a92012-04-24 22:59:51 +01004048 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
4049 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
4050 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4051 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
4052 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
4053
4054 enable_mask = ~dev_priv->irq_mask;
Ville Syrjälä21ad8332013-02-19 15:16:39 +02004055 enable_mask &= ~(I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4056 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
Chris Wilsonbbba0a92012-04-24 22:59:51 +01004057 enable_mask |= I915_USER_INTERRUPT;
4058
4059 if (IS_G4X(dev))
4060 enable_mask |= I915_BSD_USER_INTERRUPT;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004061
Daniel Vetterb79480b2013-06-27 17:52:10 +02004062 /* Interrupt setup is already guaranteed to be single-threaded, this is
4063 * just to make the assert_spin_locked check happy. */
Daniel Vetterd6207432014-09-15 14:55:27 +02004064 spin_lock_irq(&dev_priv->irq_lock);
Imre Deak755e9012014-02-10 18:42:47 +02004065 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
4066 i915_enable_pipestat(dev_priv, PIPE_A, PIPE_CRC_DONE_INTERRUPT_STATUS);
4067 i915_enable_pipestat(dev_priv, PIPE_B, PIPE_CRC_DONE_INTERRUPT_STATUS);
Daniel Vetterd6207432014-09-15 14:55:27 +02004068 spin_unlock_irq(&dev_priv->irq_lock);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004069
Chris Wilsona266c7d2012-04-24 22:59:44 +01004070 /*
4071 * Enable some error detection, note the instruction error mask
4072 * bit is reserved, so we leave it masked.
4073 */
4074 if (IS_G4X(dev)) {
4075 error_mask = ~(GM45_ERROR_PAGE_TABLE |
4076 GM45_ERROR_MEM_PRIV |
4077 GM45_ERROR_CP_PRIV |
4078 I915_ERROR_MEMORY_REFRESH);
4079 } else {
4080 error_mask = ~(I915_ERROR_PAGE_TABLE |
4081 I915_ERROR_MEMORY_REFRESH);
4082 }
4083 I915_WRITE(EMR, error_mask);
4084
4085 I915_WRITE(IMR, dev_priv->irq_mask);
4086 I915_WRITE(IER, enable_mask);
4087 POSTING_READ(IER);
4088
Daniel Vetter20afbda2012-12-11 14:05:07 +01004089 I915_WRITE(PORT_HOTPLUG_EN, 0);
4090 POSTING_READ(PORT_HOTPLUG_EN);
4091
Jani Nikulaf49e38d2013-04-29 13:02:54 +03004092 i915_enable_asle_pipestat(dev);
Daniel Vetter20afbda2012-12-11 14:05:07 +01004093
4094 return 0;
4095}
4096
Egbert Eichbac56d52013-02-25 12:06:51 -05004097static void i915_hpd_irq_setup(struct drm_device *dev)
Daniel Vetter20afbda2012-12-11 14:05:07 +01004098{
Jani Nikula2d1013d2014-03-31 14:27:17 +03004099 struct drm_i915_private *dev_priv = dev->dev_private;
Egbert Eichcd569ae2013-04-16 13:36:57 +02004100 struct intel_encoder *intel_encoder;
Daniel Vetter20afbda2012-12-11 14:05:07 +01004101 u32 hotplug_en;
4102
Daniel Vetterb5ea2d52013-06-27 17:52:15 +02004103 assert_spin_locked(&dev_priv->irq_lock);
4104
Egbert Eichbac56d52013-02-25 12:06:51 -05004105 if (I915_HAS_HOTPLUG(dev)) {
4106 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
4107 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
4108 /* Note HDMI and DP share hotplug bits */
Egbert Eiche5868a32013-02-28 04:17:12 -05004109 /* enable bits are the same for all generations */
Damien Lespiaub2784e12014-08-05 11:29:37 +01004110 for_each_intel_encoder(dev, intel_encoder)
Egbert Eichcd569ae2013-04-16 13:36:57 +02004111 if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
4112 hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
Egbert Eichbac56d52013-02-25 12:06:51 -05004113 /* Programming the CRT detection parameters tends
4114 to generate a spurious hotplug event about three
4115 seconds later. So just do it once.
4116 */
4117 if (IS_G4X(dev))
4118 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
Daniel Vetter85fc95b2013-03-27 15:47:11 +01004119 hotplug_en &= ~CRT_HOTPLUG_VOLTAGE_COMPARE_MASK;
Egbert Eichbac56d52013-02-25 12:06:51 -05004120 hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004121
Egbert Eichbac56d52013-02-25 12:06:51 -05004122 /* Ignore TV since it's buggy */
4123 I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
4124 }
Chris Wilsona266c7d2012-04-24 22:59:44 +01004125}
4126
Daniel Vetterff1f5252012-10-02 15:10:55 +02004127static irqreturn_t i965_irq_handler(int irq, void *arg)
Chris Wilsona266c7d2012-04-24 22:59:44 +01004128{
Daniel Vetter45a83f82014-05-12 19:17:55 +02004129 struct drm_device *dev = arg;
Jani Nikula2d1013d2014-03-31 14:27:17 +03004130 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004131 u32 iir, new_iir;
4132 u32 pipe_stats[I915_MAX_PIPES];
Chris Wilsona266c7d2012-04-24 22:59:44 +01004133 int ret = IRQ_NONE, pipe;
Ville Syrjälä21ad8332013-02-19 15:16:39 +02004134 u32 flip_mask =
4135 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
4136 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004137
Chris Wilsona266c7d2012-04-24 22:59:44 +01004138 iir = I915_READ(IIR);
4139
Chris Wilsona266c7d2012-04-24 22:59:44 +01004140 for (;;) {
Ville Syrjälä501e01d2014-01-17 11:35:15 +02004141 bool irq_received = (iir & ~flip_mask) != 0;
Chris Wilson2c8ba292012-04-24 22:59:46 +01004142 bool blc_event = false;
4143
Chris Wilsona266c7d2012-04-24 22:59:44 +01004144 /* Can't rely on pipestat interrupt bit in iir as it might
4145 * have been cleared after the pipestat interrupt was received.
4146 * It doesn't set the bit in iir again, but it still produces
4147 * interrupts (for non-MSI).
4148 */
Daniel Vetter222c7f52014-09-15 14:55:28 +02004149 spin_lock(&dev_priv->irq_lock);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004150 if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
Mika Kuoppala58174462014-02-25 17:11:26 +02004151 i915_handle_error(dev, false,
4152 "Command parser error, iir 0x%08x",
4153 iir);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004154
Damien Lespiau055e3932014-08-18 13:49:10 +01004155 for_each_pipe(dev_priv, pipe) {
Chris Wilsona266c7d2012-04-24 22:59:44 +01004156 int reg = PIPESTAT(pipe);
4157 pipe_stats[pipe] = I915_READ(reg);
4158
4159 /*
4160 * Clear the PIPE*STAT regs before the IIR
4161 */
4162 if (pipe_stats[pipe] & 0x8000ffff) {
Chris Wilsona266c7d2012-04-24 22:59:44 +01004163 I915_WRITE(reg, pipe_stats[pipe]);
Ville Syrjälä501e01d2014-01-17 11:35:15 +02004164 irq_received = true;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004165 }
4166 }
Daniel Vetter222c7f52014-09-15 14:55:28 +02004167 spin_unlock(&dev_priv->irq_lock);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004168
4169 if (!irq_received)
4170 break;
4171
4172 ret = IRQ_HANDLED;
4173
4174 /* Consume port. Then clear IIR or we'll miss events */
Ville Syrjälä16c6c562014-04-01 10:54:36 +03004175 if (iir & I915_DISPLAY_PORT_INTERRUPT)
4176 i9xx_hpd_irq_handler(dev);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004177
Ville Syrjälä21ad8332013-02-19 15:16:39 +02004178 I915_WRITE(IIR, iir & ~flip_mask);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004179 new_iir = I915_READ(IIR); /* Flush posted writes */
4180
Chris Wilsona266c7d2012-04-24 22:59:44 +01004181 if (iir & I915_USER_INTERRUPT)
4182 notify_ring(dev, &dev_priv->ring[RCS]);
4183 if (iir & I915_BSD_USER_INTERRUPT)
4184 notify_ring(dev, &dev_priv->ring[VCS]);
4185
Damien Lespiau055e3932014-08-18 13:49:10 +01004186 for_each_pipe(dev_priv, pipe) {
Chris Wilson2c8ba292012-04-24 22:59:46 +01004187 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
Ville Syrjälä90a72f82013-02-19 23:16:44 +02004188 i915_handle_vblank(dev, pipe, pipe, iir))
4189 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004190
4191 if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
4192 blc_event = true;
Daniel Vetter4356d582013-10-16 22:55:55 +02004193
4194 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
Daniel Vetter277de952013-10-18 16:37:07 +02004195 i9xx_pipe_crc_irq_handler(dev, pipe);
Chris Wilsona266c7d2012-04-24 22:59:44 +01004196
Daniel Vetter1f7247c2014-09-30 10:56:48 +02004197 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
4198 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
Ville Syrjälä2d9d2b02014-01-17 11:44:31 +02004199 }
Chris Wilsona266c7d2012-04-24 22:59:44 +01004200
4201 if (blc_event || (iir & I915_ASLE_INTERRUPT))
4202 intel_opregion_asle_intr(dev);
4203
Daniel Vetter515ac2b2012-12-01 13:53:44 +01004204 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
4205 gmbus_irq_handler(dev);
4206
Chris Wilsona266c7d2012-04-24 22:59:44 +01004207 /* With MSI, interrupts are only generated when iir
4208 * transitions from zero to nonzero. If another bit got
4209 * set while we were handling the existing iir bits, then
4210 * we would never get another interrupt.
4211 *
4212 * This is fine on non-MSI as well, as if we hit this path
4213 * we avoid exiting the interrupt handler only to generate
4214 * another one.
4215 *
4216 * Note that for MSI this could cause a stray interrupt report
4217 * if an interrupt landed in the time between writing IIR and
4218 * the posting read. This should be rare enough to never
4219 * trigger the 99% of 100,000 interrupts test for disabling
4220 * stray interrupts.
4221 */
4222 iir = new_iir;
4223 }
4224
Daniel Vetterd05c6172012-04-26 23:28:09 +02004225 i915_update_dri1_breadcrumb(dev);
Chris Wilson2c8ba292012-04-24 22:59:46 +01004226
Chris Wilsona266c7d2012-04-24 22:59:44 +01004227 return ret;
4228}
4229
4230static void i965_irq_uninstall(struct drm_device * dev)
4231{
Jani Nikula2d1013d2014-03-31 14:27:17 +03004232 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilsona266c7d2012-04-24 22:59:44 +01004233 int pipe;
4234
4235 if (!dev_priv)
4236 return;
4237
Chris Wilsonadca4732012-05-11 18:01:31 +01004238 I915_WRITE(PORT_HOTPLUG_EN, 0);
4239 I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
Chris Wilsona266c7d2012-04-24 22:59:44 +01004240
4241 I915_WRITE(HWSTAM, 0xffffffff);
Damien Lespiau055e3932014-08-18 13:49:10 +01004242 for_each_pipe(dev_priv, pipe)
Chris Wilsona266c7d2012-04-24 22:59:44 +01004243 I915_WRITE(PIPESTAT(pipe), 0);
4244 I915_WRITE(IMR, 0xffffffff);
4245 I915_WRITE(IER, 0x0);
4246
Damien Lespiau055e3932014-08-18 13:49:10 +01004247 for_each_pipe(dev_priv, pipe)
Chris Wilsona266c7d2012-04-24 22:59:44 +01004248 I915_WRITE(PIPESTAT(pipe),
4249 I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
4250 I915_WRITE(IIR, I915_READ(IIR));
4251}
4252
Daniel Vetter4cb21832014-09-15 14:55:26 +02004253static void intel_hpd_irq_reenable_work(struct work_struct *work)
Egbert Eichac4c16c2013-04-16 13:36:58 +02004254{
Imre Deak63237512014-08-18 15:37:02 +03004255 struct drm_i915_private *dev_priv =
4256 container_of(work, typeof(*dev_priv),
4257 hotplug_reenable_work.work);
Egbert Eichac4c16c2013-04-16 13:36:58 +02004258 struct drm_device *dev = dev_priv->dev;
4259 struct drm_mode_config *mode_config = &dev->mode_config;
Egbert Eichac4c16c2013-04-16 13:36:58 +02004260 int i;
4261
Imre Deak63237512014-08-18 15:37:02 +03004262 intel_runtime_pm_get(dev_priv);
4263
Daniel Vetter4cb21832014-09-15 14:55:26 +02004264 spin_lock_irq(&dev_priv->irq_lock);
Egbert Eichac4c16c2013-04-16 13:36:58 +02004265 for (i = (HPD_NONE + 1); i < HPD_NUM_PINS; i++) {
4266 struct drm_connector *connector;
4267
4268 if (dev_priv->hpd_stats[i].hpd_mark != HPD_DISABLED)
4269 continue;
4270
4271 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
4272
4273 list_for_each_entry(connector, &mode_config->connector_list, head) {
4274 struct intel_connector *intel_connector = to_intel_connector(connector);
4275
4276 if (intel_connector->encoder->hpd_pin == i) {
4277 if (connector->polled != intel_connector->polled)
4278 DRM_DEBUG_DRIVER("Reenabling HPD on connector %s\n",
Jani Nikulac23cc412014-06-03 14:56:17 +03004279 connector->name);
Egbert Eichac4c16c2013-04-16 13:36:58 +02004280 connector->polled = intel_connector->polled;
4281 if (!connector->polled)
4282 connector->polled = DRM_CONNECTOR_POLL_HPD;
4283 }
4284 }
4285 }
4286 if (dev_priv->display.hpd_irq_setup)
4287 dev_priv->display.hpd_irq_setup(dev);
Daniel Vetter4cb21832014-09-15 14:55:26 +02004288 spin_unlock_irq(&dev_priv->irq_lock);
Imre Deak63237512014-08-18 15:37:02 +03004289
4290 intel_runtime_pm_put(dev_priv);
Egbert Eichac4c16c2013-04-16 13:36:58 +02004291}
4292
Daniel Vetterfca52a52014-09-30 10:56:45 +02004293/**
4294 * intel_irq_init - initializes irq support
4295 * @dev_priv: i915 device instance
4296 *
4297 * This function initializes all the irq support including work items, timers
4298 * and all the vtables. It does not setup the interrupt itself though.
4299 */
Daniel Vetterb9632912014-09-30 10:56:44 +02004300void intel_irq_init(struct drm_i915_private *dev_priv)
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004301{
Daniel Vetterb9632912014-09-30 10:56:44 +02004302 struct drm_device *dev = dev_priv->dev;
Chris Wilson8b2e3262012-04-24 22:59:41 +01004303
4304 INIT_WORK(&dev_priv->hotplug_work, i915_hotplug_work_func);
Dave Airlie13cf5502014-06-18 11:29:35 +10004305 INIT_WORK(&dev_priv->dig_port_work, i915_digport_work_func);
Daniel Vetter99584db2012-11-14 17:14:04 +01004306 INIT_WORK(&dev_priv->gpu_error.work, i915_error_work_func);
Daniel Vetterc6a828d2012-08-08 23:35:35 +02004307 INIT_WORK(&dev_priv->rps.work, gen6_pm_rps_work);
Daniel Vettera4da4fa2012-11-02 19:55:07 +01004308 INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
Chris Wilson8b2e3262012-04-24 22:59:41 +01004309
Deepak Sa6706b42014-03-15 20:23:22 +05304310 /* Let's track the enabled rps events */
Daniel Vetterb9632912014-09-30 10:56:44 +02004311 if (IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
Ville Syrjälä6c65a582014-08-29 14:14:07 +03004312 /* WaGsvRC0ResidencyMethod:vlv */
Deepak S31685c22014-07-03 17:33:01 -04004313 dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
4314 else
4315 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
Deepak Sa6706b42014-03-15 20:23:22 +05304316
Daniel Vetter99584db2012-11-14 17:14:04 +01004317 setup_timer(&dev_priv->gpu_error.hangcheck_timer,
4318 i915_hangcheck_elapsed,
Daniel Vetter61bac782012-12-01 21:03:21 +01004319 (unsigned long) dev);
Imre Deak63237512014-08-18 15:37:02 +03004320 INIT_DELAYED_WORK(&dev_priv->hotplug_reenable_work,
Daniel Vetter4cb21832014-09-15 14:55:26 +02004321 intel_hpd_irq_reenable_work);
Daniel Vetter61bac782012-12-01 21:03:21 +01004322
Tomas Janousek97a19a22012-12-08 13:48:13 +01004323 pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY, PM_QOS_DEFAULT_VALUE);
Daniel Vetter9ee32fea2012-12-01 13:53:48 +01004324
Daniel Vetterb9632912014-09-30 10:56:44 +02004325 if (IS_GEN2(dev_priv)) {
Ville Syrjälä4cdb83e2013-10-11 21:52:44 +03004326 dev->max_vblank_count = 0;
4327 dev->driver->get_vblank_counter = i8xx_get_vblank_counter;
Daniel Vetterb9632912014-09-30 10:56:44 +02004328 } else if (IS_G4X(dev_priv) || INTEL_INFO(dev_priv)->gen >= 5) {
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004329 dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
4330 dev->driver->get_vblank_counter = gm45_get_vblank_counter;
Ville Syrjälä391f75e2013-09-25 19:55:26 +03004331 } else {
4332 dev->driver->get_vblank_counter = i915_get_vblank_counter;
4333 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004334 }
4335
Ville Syrjälä21da2702014-08-06 14:49:55 +03004336 /*
4337 * Opt out of the vblank disable timer on everything except gen2.
4338 * Gen2 doesn't have a hardware frame counter and so depends on
4339 * vblank interrupts to produce sane vblank seuquence numbers.
4340 */
Daniel Vetterb9632912014-09-30 10:56:44 +02004341 if (!IS_GEN2(dev_priv))
Ville Syrjälä21da2702014-08-06 14:49:55 +03004342 dev->vblank_disable_immediate = true;
4343
Ville Syrjäläc2baf4b2013-09-23 14:48:50 +03004344 if (drm_core_check_feature(dev, DRIVER_MODESET)) {
Keith Packardc3613de2011-08-12 17:05:54 -07004345 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
Ville Syrjäläc2baf4b2013-09-23 14:48:50 +03004346 dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
4347 }
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004348
Daniel Vetterb9632912014-09-30 10:56:44 +02004349 if (IS_CHERRYVIEW(dev_priv)) {
Ville Syrjälä43f328d2014-04-09 20:40:52 +03004350 dev->driver->irq_handler = cherryview_irq_handler;
4351 dev->driver->irq_preinstall = cherryview_irq_preinstall;
4352 dev->driver->irq_postinstall = cherryview_irq_postinstall;
4353 dev->driver->irq_uninstall = cherryview_irq_uninstall;
4354 dev->driver->enable_vblank = valleyview_enable_vblank;
4355 dev->driver->disable_vblank = valleyview_disable_vblank;
4356 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
Daniel Vetterb9632912014-09-30 10:56:44 +02004357 } else if (IS_VALLEYVIEW(dev_priv)) {
Jesse Barnes7e231dbe2012-03-28 13:39:38 -07004358 dev->driver->irq_handler = valleyview_irq_handler;
4359 dev->driver->irq_preinstall = valleyview_irq_preinstall;
4360 dev->driver->irq_postinstall = valleyview_irq_postinstall;
4361 dev->driver->irq_uninstall = valleyview_irq_uninstall;
4362 dev->driver->enable_vblank = valleyview_enable_vblank;
4363 dev->driver->disable_vblank = valleyview_disable_vblank;
Egbert Eichfa00abe2013-02-25 12:06:48 -05004364 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
Daniel Vetterb9632912014-09-30 10:56:44 +02004365 } else if (INTEL_INFO(dev_priv)->gen >= 8) {
Ben Widawskyabd58f02013-11-02 21:07:09 -07004366 dev->driver->irq_handler = gen8_irq_handler;
Daniel Vetter723761b2014-05-22 17:56:34 +02004367 dev->driver->irq_preinstall = gen8_irq_reset;
Ben Widawskyabd58f02013-11-02 21:07:09 -07004368 dev->driver->irq_postinstall = gen8_irq_postinstall;
4369 dev->driver->irq_uninstall = gen8_irq_uninstall;
4370 dev->driver->enable_vblank = gen8_enable_vblank;
4371 dev->driver->disable_vblank = gen8_disable_vblank;
4372 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004373 } else if (HAS_PCH_SPLIT(dev)) {
4374 dev->driver->irq_handler = ironlake_irq_handler;
Daniel Vetter723761b2014-05-22 17:56:34 +02004375 dev->driver->irq_preinstall = ironlake_irq_reset;
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004376 dev->driver->irq_postinstall = ironlake_irq_postinstall;
4377 dev->driver->irq_uninstall = ironlake_irq_uninstall;
4378 dev->driver->enable_vblank = ironlake_enable_vblank;
4379 dev->driver->disable_vblank = ironlake_disable_vblank;
Daniel Vetter82a28bc2013-03-27 15:55:01 +01004380 dev_priv->display.hpd_irq_setup = ibx_hpd_irq_setup;
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004381 } else {
Daniel Vetterb9632912014-09-30 10:56:44 +02004382 if (INTEL_INFO(dev_priv)->gen == 2) {
Chris Wilsonc2798b12012-04-22 21:13:57 +01004383 dev->driver->irq_preinstall = i8xx_irq_preinstall;
4384 dev->driver->irq_postinstall = i8xx_irq_postinstall;
4385 dev->driver->irq_handler = i8xx_irq_handler;
4386 dev->driver->irq_uninstall = i8xx_irq_uninstall;
Daniel Vetterb9632912014-09-30 10:56:44 +02004387 } else if (INTEL_INFO(dev_priv)->gen == 3) {
Chris Wilsona266c7d2012-04-24 22:59:44 +01004388 dev->driver->irq_preinstall = i915_irq_preinstall;
4389 dev->driver->irq_postinstall = i915_irq_postinstall;
4390 dev->driver->irq_uninstall = i915_irq_uninstall;
4391 dev->driver->irq_handler = i915_irq_handler;
Daniel Vetter20afbda2012-12-11 14:05:07 +01004392 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
Chris Wilsonc2798b12012-04-22 21:13:57 +01004393 } else {
Chris Wilsona266c7d2012-04-24 22:59:44 +01004394 dev->driver->irq_preinstall = i965_irq_preinstall;
4395 dev->driver->irq_postinstall = i965_irq_postinstall;
4396 dev->driver->irq_uninstall = i965_irq_uninstall;
4397 dev->driver->irq_handler = i965_irq_handler;
Egbert Eichbac56d52013-02-25 12:06:51 -05004398 dev_priv->display.hpd_irq_setup = i915_hpd_irq_setup;
Chris Wilsonc2798b12012-04-22 21:13:57 +01004399 }
Jesse Barnesf71d4af2011-06-28 13:00:41 -07004400 dev->driver->enable_vblank = i915_enable_vblank;
4401 dev->driver->disable_vblank = i915_disable_vblank;
4402 }
4403}
Daniel Vetter20afbda2012-12-11 14:05:07 +01004404
Daniel Vetterfca52a52014-09-30 10:56:45 +02004405/**
4406 * intel_hpd_init - initializes and enables hpd support
4407 * @dev_priv: i915 device instance
4408 *
4409 * This function enables the hotplug support. It requires that interrupts have
4410 * already been enabled with intel_irq_init_hw(). From this point on hotplug and
4411 * poll request can run concurrently to other code, so locking rules must be
4412 * obeyed.
4413 *
4414 * This is a separate step from interrupt enabling to simplify the locking rules
4415 * in the driver load and resume code.
4416 */
Daniel Vetterb9632912014-09-30 10:56:44 +02004417void intel_hpd_init(struct drm_i915_private *dev_priv)
Daniel Vetter20afbda2012-12-11 14:05:07 +01004418{
Daniel Vetterb9632912014-09-30 10:56:44 +02004419 struct drm_device *dev = dev_priv->dev;
Egbert Eich821450c2013-04-16 13:36:55 +02004420 struct drm_mode_config *mode_config = &dev->mode_config;
4421 struct drm_connector *connector;
4422 int i;
Daniel Vetter20afbda2012-12-11 14:05:07 +01004423
Egbert Eich821450c2013-04-16 13:36:55 +02004424 for (i = 1; i < HPD_NUM_PINS; i++) {
4425 dev_priv->hpd_stats[i].hpd_cnt = 0;
4426 dev_priv->hpd_stats[i].hpd_mark = HPD_ENABLED;
4427 }
4428 list_for_each_entry(connector, &mode_config->connector_list, head) {
4429 struct intel_connector *intel_connector = to_intel_connector(connector);
4430 connector->polled = intel_connector->polled;
Dave Airlie0e32b392014-05-02 14:02:48 +10004431 if (connector->encoder && !connector->polled && I915_HAS_HOTPLUG(dev) && intel_connector->encoder->hpd_pin > HPD_NONE)
4432 connector->polled = DRM_CONNECTOR_POLL_HPD;
4433 if (intel_connector->mst_port)
Egbert Eich821450c2013-04-16 13:36:55 +02004434 connector->polled = DRM_CONNECTOR_POLL_HPD;
4435 }
Daniel Vetterb5ea2d52013-06-27 17:52:15 +02004436
4437 /* Interrupt setup is already guaranteed to be single-threaded, this is
4438 * just to make the assert_spin_locked checks happy. */
Daniel Vetterd6207432014-09-15 14:55:27 +02004439 spin_lock_irq(&dev_priv->irq_lock);
Daniel Vetter20afbda2012-12-11 14:05:07 +01004440 if (dev_priv->display.hpd_irq_setup)
4441 dev_priv->display.hpd_irq_setup(dev);
Daniel Vetterd6207432014-09-15 14:55:27 +02004442 spin_unlock_irq(&dev_priv->irq_lock);
Daniel Vetter20afbda2012-12-11 14:05:07 +01004443}
Paulo Zanonic67a4702013-08-19 13:18:09 -03004444
Daniel Vetterfca52a52014-09-30 10:56:45 +02004445/**
4446 * intel_irq_install - enables the hardware interrupt
4447 * @dev_priv: i915 device instance
4448 *
4449 * This function enables the hardware interrupt handling, but leaves the hotplug
4450 * handling still disabled. It is called after intel_irq_init().
4451 *
4452 * In the driver load and resume code we need working interrupts in a few places
4453 * but don't want to deal with the hassle of concurrent probe and hotplug
4454 * workers. Hence the split into this two-stage approach.
4455 */
Daniel Vetter2aeb7d32014-09-30 10:56:43 +02004456int intel_irq_install(struct drm_i915_private *dev_priv)
4457{
4458 /*
4459 * We enable some interrupt sources in our postinstall hooks, so mark
4460 * interrupts as enabled _before_ actually enabling them to avoid
4461 * special cases in our ordering checks.
4462 */
4463 dev_priv->pm.irqs_enabled = true;
4464
4465 return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
4466}
4467
Daniel Vetterfca52a52014-09-30 10:56:45 +02004468/**
4469 * intel_irq_uninstall - finilizes all irq handling
4470 * @dev_priv: i915 device instance
4471 *
4472 * This stops interrupt and hotplug handling and unregisters and frees all
4473 * resources acquired in the init functions.
4474 */
Daniel Vetter2aeb7d32014-09-30 10:56:43 +02004475void intel_irq_uninstall(struct drm_i915_private *dev_priv)
4476{
4477 drm_irq_uninstall(dev_priv->dev);
4478 intel_hpd_cancel_work(dev_priv);
4479 dev_priv->pm.irqs_enabled = false;
4480}
4481
Daniel Vetterfca52a52014-09-30 10:56:45 +02004482/**
4483 * intel_runtime_pm_disable_interrupts - runtime interrupt disabling
4484 * @dev_priv: i915 device instance
4485 *
4486 * This function is used to disable interrupts at runtime, both in the runtime
4487 * pm and the system suspend/resume code.
4488 */
Daniel Vetterb9632912014-09-30 10:56:44 +02004489void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
Paulo Zanonic67a4702013-08-19 13:18:09 -03004490{
Daniel Vetterb9632912014-09-30 10:56:44 +02004491 dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
Daniel Vetter2aeb7d32014-09-30 10:56:43 +02004492 dev_priv->pm.irqs_enabled = false;
Paulo Zanonic67a4702013-08-19 13:18:09 -03004493}
4494
Daniel Vetterfca52a52014-09-30 10:56:45 +02004495/**
4496 * intel_runtime_pm_enable_interrupts - runtime interrupt enabling
4497 * @dev_priv: i915 device instance
4498 *
4499 * This function is used to enable interrupts at runtime, both in the runtime
4500 * pm and the system suspend/resume code.
4501 */
Daniel Vetterb9632912014-09-30 10:56:44 +02004502void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
Paulo Zanonic67a4702013-08-19 13:18:09 -03004503{
Daniel Vetter2aeb7d32014-09-30 10:56:43 +02004504 dev_priv->pm.irqs_enabled = true;
Daniel Vetterb9632912014-09-30 10:56:44 +02004505 dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
4506 dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
Paulo Zanonic67a4702013-08-19 13:18:09 -03004507}