blob: a69dc19a0bdba760e936ae5fd292054197480034 [file] [log] [blame]
Eric Anholt673a3942008-07-30 12:06:12 -07001/*
Daniel Vetterbe6a0372015-03-18 10:46:04 +01002 * Copyright © 2008-2015 Intel Corporation
Eric Anholt673a3942008-07-30 12:06:12 -07003 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
David Howells760285e2012-10-02 18:01:07 +010028#include <drm/drmP.h>
David Herrmann0de23972013-07-24 21:07:52 +020029#include <drm/drm_vma_manager.h>
David Howells760285e2012-10-02 18:01:07 +010030#include <drm/i915_drm.h>
Eric Anholt673a3942008-07-30 12:06:12 -070031#include "i915_drv.h"
Chris Wilson57822dc2017-02-22 11:40:48 +000032#include "i915_gem_clflush.h"
Yu Zhangeb822892015-02-10 19:05:49 +080033#include "i915_vgpu.h"
Chris Wilson1c5d22f2009-08-25 11:15:50 +010034#include "i915_trace.h"
Jesse Barnes652c3932009-08-17 13:31:43 -070035#include "intel_drv.h"
Chris Wilson5d723d72016-08-04 16:32:35 +010036#include "intel_frontbuffer.h"
Peter Antoine0ccdacf2016-04-13 15:03:25 +010037#include "intel_mocs.h"
Matthew Auld465c4032017-10-06 23:18:14 +010038#include "i915_gemfs.h"
Chris Wilson6b5e90f2016-11-14 20:41:05 +000039#include <linux/dma-fence-array.h>
Chris Wilsonfe3288b2017-02-12 17:20:01 +000040#include <linux/kthread.h>
Chris Wilsonc13d87e2016-07-20 09:21:15 +010041#include <linux/reservation.h>
Hugh Dickins5949eac2011-06-27 16:18:18 -070042#include <linux/shmem_fs.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090043#include <linux/slab.h>
Chris Wilson20e49332016-11-22 14:41:21 +000044#include <linux/stop_machine.h>
Eric Anholt673a3942008-07-30 12:06:12 -070045#include <linux/swap.h>
Jesse Barnes79e53942008-11-07 14:24:08 -080046#include <linux/pci.h>
Daniel Vetter1286ff72012-05-10 15:25:09 +020047#include <linux/dma-buf.h>
Eric Anholt673a3942008-07-30 12:06:12 -070048
Chris Wilsonfbbd37b2016-10-28 13:58:42 +010049static void i915_gem_flush_free_objects(struct drm_i915_private *i915);
Chris Wilson61050802012-04-17 15:31:31 +010050
Chris Wilson2c225692013-08-09 12:26:45 +010051static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
52{
Chris Wilsone27ab732017-06-15 13:38:49 +010053 if (obj->cache_dirty)
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +053054 return false;
55
Chris Wilsonb8f55be2017-08-11 12:11:16 +010056 if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))
Chris Wilson2c225692013-08-09 12:26:45 +010057 return true;
58
Chris Wilsonbd3d2252017-10-13 21:26:14 +010059 return obj->pin_global; /* currently in use by HW, keep flushed */
Chris Wilson2c225692013-08-09 12:26:45 +010060}
61
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +053062static int
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +010063insert_mappable_node(struct i915_ggtt *ggtt,
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +053064 struct drm_mm_node *node, u32 size)
65{
66 memset(node, 0, sizeof(*node));
Chris Wilson4e64e552017-02-02 21:04:38 +000067 return drm_mm_insert_node_in_range(&ggtt->base.mm, node,
68 size, 0, I915_COLOR_UNEVICTABLE,
69 0, ggtt->mappable_end,
70 DRM_MM_INSERT_LOW);
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +053071}
72
73static void
74remove_mappable_node(struct drm_mm_node *node)
75{
76 drm_mm_remove_node(node);
77}
78
Chris Wilson73aa8082010-09-30 11:46:12 +010079/* some bookkeeping */
80static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
Chris Wilson3ef7f222016-10-18 13:02:48 +010081 u64 size)
Chris Wilson73aa8082010-09-30 11:46:12 +010082{
Daniel Vetterc20e8352013-07-24 22:40:23 +020083 spin_lock(&dev_priv->mm.object_stat_lock);
Chris Wilson73aa8082010-09-30 11:46:12 +010084 dev_priv->mm.object_count++;
85 dev_priv->mm.object_memory += size;
Daniel Vetterc20e8352013-07-24 22:40:23 +020086 spin_unlock(&dev_priv->mm.object_stat_lock);
Chris Wilson73aa8082010-09-30 11:46:12 +010087}
88
89static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
Chris Wilson3ef7f222016-10-18 13:02:48 +010090 u64 size)
Chris Wilson73aa8082010-09-30 11:46:12 +010091{
Daniel Vetterc20e8352013-07-24 22:40:23 +020092 spin_lock(&dev_priv->mm.object_stat_lock);
Chris Wilson73aa8082010-09-30 11:46:12 +010093 dev_priv->mm.object_count--;
94 dev_priv->mm.object_memory -= size;
Daniel Vetterc20e8352013-07-24 22:40:23 +020095 spin_unlock(&dev_priv->mm.object_stat_lock);
Chris Wilson73aa8082010-09-30 11:46:12 +010096}
97
Chris Wilson21dd3732011-01-26 15:55:56 +000098static int
Daniel Vetter33196de2012-11-14 17:14:05 +010099i915_gem_wait_for_error(struct i915_gpu_error *error)
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100100{
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100101 int ret;
102
Chris Wilson4c7d62c2016-10-28 13:58:32 +0100103 might_sleep();
104
Daniel Vetter0a6759c2012-07-04 22:18:41 +0200105 /*
106 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
107 * userspace. If it takes that long something really bad is going on and
108 * we should simply try to bail out and fail as gracefully as possible.
109 */
Daniel Vetter1f83fee2012-11-15 17:17:22 +0100110 ret = wait_event_interruptible_timeout(error->reset_queue,
Chris Wilson8c185ec2017-03-16 17:13:02 +0000111 !i915_reset_backoff(error),
Chris Wilsonb52992c2016-10-28 13:58:24 +0100112 I915_RESET_TIMEOUT);
Daniel Vetter0a6759c2012-07-04 22:18:41 +0200113 if (ret == 0) {
114 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
115 return -EIO;
116 } else if (ret < 0) {
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100117 return ret;
Chris Wilsond98c52c2016-04-13 17:35:05 +0100118 } else {
119 return 0;
Daniel Vetter0a6759c2012-07-04 22:18:41 +0200120 }
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100121}
122
Chris Wilson54cf91d2010-11-25 18:00:26 +0000123int i915_mutex_lock_interruptible(struct drm_device *dev)
Chris Wilson76c1dec2010-09-25 11:22:51 +0100124{
Chris Wilsonfac5e232016-07-04 11:34:36 +0100125 struct drm_i915_private *dev_priv = to_i915(dev);
Chris Wilson76c1dec2010-09-25 11:22:51 +0100126 int ret;
127
Daniel Vetter33196de2012-11-14 17:14:05 +0100128 ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
Chris Wilson76c1dec2010-09-25 11:22:51 +0100129 if (ret)
130 return ret;
131
132 ret = mutex_lock_interruptible(&dev->struct_mutex);
133 if (ret)
134 return ret;
135
Chris Wilson76c1dec2010-09-25 11:22:51 +0100136 return 0;
137}
Chris Wilson30dbf0c2010-09-25 10:19:17 +0100138
Chris Wilsone4d20062018-04-06 16:51:44 +0100139static u32 __i915_gem_park(struct drm_i915_private *i915)
140{
141 lockdep_assert_held(&i915->drm.struct_mutex);
142 GEM_BUG_ON(i915->gt.active_requests);
143
144 if (!i915->gt.awake)
145 return I915_EPOCH_INVALID;
146
147 GEM_BUG_ON(i915->gt.epoch == I915_EPOCH_INVALID);
148
149 /*
150 * Be paranoid and flush a concurrent interrupt to make sure
151 * we don't reactivate any irq tasklets after parking.
152 *
153 * FIXME: Note that even though we have waited for execlists to be idle,
154 * there may still be an in-flight interrupt even though the CSB
155 * is now empty. synchronize_irq() makes sure that a residual interrupt
156 * is completed before we continue, but it doesn't prevent the HW from
157 * raising a spurious interrupt later. To complete the shield we should
158 * coordinate disabling the CS irq with flushing the interrupts.
159 */
160 synchronize_irq(i915->drm.irq);
161
162 intel_engines_park(i915);
163 i915_gem_timelines_park(i915);
164
165 i915_pmu_gt_parked(i915);
166
167 i915->gt.awake = false;
168
169 if (INTEL_GEN(i915) >= 6)
170 gen6_rps_idle(i915);
171
172 intel_display_power_put(i915, POWER_DOMAIN_GT_IRQ);
173
174 intel_runtime_pm_put(i915);
175
176 return i915->gt.epoch;
177}
178
179void i915_gem_park(struct drm_i915_private *i915)
180{
181 lockdep_assert_held(&i915->drm.struct_mutex);
182 GEM_BUG_ON(i915->gt.active_requests);
183
184 if (!i915->gt.awake)
185 return;
186
187 /* Defer the actual call to __i915_gem_park() to prevent ping-pongs */
188 mod_delayed_work(i915->wq, &i915->gt.idle_work, msecs_to_jiffies(100));
189}
190
191void i915_gem_unpark(struct drm_i915_private *i915)
192{
193 lockdep_assert_held(&i915->drm.struct_mutex);
194 GEM_BUG_ON(!i915->gt.active_requests);
195
196 if (i915->gt.awake)
197 return;
198
199 intel_runtime_pm_get_noresume(i915);
200
201 /*
202 * It seems that the DMC likes to transition between the DC states a lot
203 * when there are no connected displays (no active power domains) during
204 * command submission.
205 *
206 * This activity has negative impact on the performance of the chip with
207 * huge latencies observed in the interrupt handler and elsewhere.
208 *
209 * Work around it by grabbing a GT IRQ power domain whilst there is any
210 * GT activity, preventing any DC state transitions.
211 */
212 intel_display_power_get(i915, POWER_DOMAIN_GT_IRQ);
213
214 i915->gt.awake = true;
215 if (unlikely(++i915->gt.epoch == 0)) /* keep 0 as invalid */
216 i915->gt.epoch = 1;
217
218 intel_enable_gt_powersave(i915);
219 i915_update_gfx_val(i915);
220 if (INTEL_GEN(i915) >= 6)
221 gen6_rps_busy(i915);
222 i915_pmu_gt_unparked(i915);
223
224 intel_engines_unpark(i915);
225
226 i915_queue_hangcheck(i915);
227
228 queue_delayed_work(i915->wq,
229 &i915->gt.retire_work,
230 round_jiffies_up_relative(HZ));
231}
232
Eric Anholt673a3942008-07-30 12:06:12 -0700233int
Eric Anholt5a125c32008-10-22 21:40:13 -0700234i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +0000235 struct drm_file *file)
Eric Anholt5a125c32008-10-22 21:40:13 -0700236{
Joonas Lahtinen72e96d62016-03-30 16:57:10 +0300237 struct drm_i915_private *dev_priv = to_i915(dev);
Joonas Lahtinen62106b42016-03-18 10:42:57 +0200238 struct i915_ggtt *ggtt = &dev_priv->ggtt;
Joonas Lahtinen72e96d62016-03-30 16:57:10 +0300239 struct drm_i915_gem_get_aperture *args = data;
Tvrtko Ursulinca1543b2015-07-01 11:51:10 +0100240 struct i915_vma *vma;
Weinan Liff8f7972017-05-31 10:35:52 +0800241 u64 pinned;
Eric Anholt5a125c32008-10-22 21:40:13 -0700242
Weinan Liff8f7972017-05-31 10:35:52 +0800243 pinned = ggtt->base.reserved;
Chris Wilson73aa8082010-09-30 11:46:12 +0100244 mutex_lock(&dev->struct_mutex);
Chris Wilson1c7f4bc2016-02-26 11:03:19 +0000245 list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
Chris Wilson20dfbde2016-08-04 16:32:30 +0100246 if (i915_vma_is_pinned(vma))
Tvrtko Ursulinca1543b2015-07-01 11:51:10 +0100247 pinned += vma->node.size;
Chris Wilson1c7f4bc2016-02-26 11:03:19 +0000248 list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
Chris Wilson20dfbde2016-08-04 16:32:30 +0100249 if (i915_vma_is_pinned(vma))
Tvrtko Ursulinca1543b2015-07-01 11:51:10 +0100250 pinned += vma->node.size;
Chris Wilson73aa8082010-09-30 11:46:12 +0100251 mutex_unlock(&dev->struct_mutex);
Eric Anholt5a125c32008-10-22 21:40:13 -0700252
Joonas Lahtinen72e96d62016-03-30 16:57:10 +0300253 args->aper_size = ggtt->base.total;
Akshay Joshi0206e352011-08-16 15:34:10 -0400254 args->aper_available_size = args->aper_size - pinned;
Chris Wilson6299f992010-11-24 12:23:44 +0000255
Eric Anholt5a125c32008-10-22 21:40:13 -0700256 return 0;
257}
258
Matthew Auldb91b09e2017-10-06 23:18:17 +0100259static int i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
Chris Wilson00731152014-05-21 12:42:56 +0100260{
Al Viro93c76a32015-12-04 23:45:44 -0500261 struct address_space *mapping = obj->base.filp->f_mapping;
Chris Wilsondbb43512016-12-07 13:34:11 +0000262 drm_dma_handle_t *phys;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800263 struct sg_table *st;
264 struct scatterlist *sg;
Chris Wilsondbb43512016-12-07 13:34:11 +0000265 char *vaddr;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800266 int i;
Matthew Auldb91b09e2017-10-06 23:18:17 +0100267 int err;
Chris Wilson00731152014-05-21 12:42:56 +0100268
Chris Wilson6a2c4232014-11-04 04:51:40 -0800269 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
Matthew Auldb91b09e2017-10-06 23:18:17 +0100270 return -EINVAL;
Chris Wilson00731152014-05-21 12:42:56 +0100271
Chris Wilsondbb43512016-12-07 13:34:11 +0000272 /* Always aligning to the object size, allows a single allocation
273 * to handle all possible callers, and given typical object sizes,
274 * the alignment of the buddy allocation will naturally match.
275 */
276 phys = drm_pci_alloc(obj->base.dev,
Ville Syrjälä750fae22017-09-07 17:32:03 +0300277 roundup_pow_of_two(obj->base.size),
Chris Wilsondbb43512016-12-07 13:34:11 +0000278 roundup_pow_of_two(obj->base.size));
279 if (!phys)
Matthew Auldb91b09e2017-10-06 23:18:17 +0100280 return -ENOMEM;
Chris Wilsondbb43512016-12-07 13:34:11 +0000281
282 vaddr = phys->vaddr;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800283 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
284 struct page *page;
285 char *src;
286
287 page = shmem_read_mapping_page(mapping, i);
Chris Wilsondbb43512016-12-07 13:34:11 +0000288 if (IS_ERR(page)) {
Matthew Auldb91b09e2017-10-06 23:18:17 +0100289 err = PTR_ERR(page);
Chris Wilsondbb43512016-12-07 13:34:11 +0000290 goto err_phys;
291 }
Chris Wilson6a2c4232014-11-04 04:51:40 -0800292
293 src = kmap_atomic(page);
294 memcpy(vaddr, src, PAGE_SIZE);
295 drm_clflush_virt_range(vaddr, PAGE_SIZE);
296 kunmap_atomic(src);
297
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300298 put_page(page);
Chris Wilson6a2c4232014-11-04 04:51:40 -0800299 vaddr += PAGE_SIZE;
300 }
301
Chris Wilsonc0336662016-05-06 15:40:21 +0100302 i915_gem_chipset_flush(to_i915(obj->base.dev));
Chris Wilson6a2c4232014-11-04 04:51:40 -0800303
304 st = kmalloc(sizeof(*st), GFP_KERNEL);
Chris Wilsondbb43512016-12-07 13:34:11 +0000305 if (!st) {
Matthew Auldb91b09e2017-10-06 23:18:17 +0100306 err = -ENOMEM;
Chris Wilsondbb43512016-12-07 13:34:11 +0000307 goto err_phys;
308 }
Chris Wilson6a2c4232014-11-04 04:51:40 -0800309
310 if (sg_alloc_table(st, 1, GFP_KERNEL)) {
311 kfree(st);
Matthew Auldb91b09e2017-10-06 23:18:17 +0100312 err = -ENOMEM;
Chris Wilsondbb43512016-12-07 13:34:11 +0000313 goto err_phys;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800314 }
315
316 sg = st->sgl;
317 sg->offset = 0;
318 sg->length = obj->base.size;
319
Chris Wilsondbb43512016-12-07 13:34:11 +0000320 sg_dma_address(sg) = phys->busaddr;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800321 sg_dma_len(sg) = obj->base.size;
322
Chris Wilsondbb43512016-12-07 13:34:11 +0000323 obj->phys_handle = phys;
Matthew Auldb91b09e2017-10-06 23:18:17 +0100324
Matthew Aulda5c081662017-10-06 23:18:18 +0100325 __i915_gem_object_set_pages(obj, st, sg->length);
Matthew Auldb91b09e2017-10-06 23:18:17 +0100326
327 return 0;
Chris Wilsondbb43512016-12-07 13:34:11 +0000328
329err_phys:
330 drm_pci_free(obj->base.dev, phys);
Matthew Auldb91b09e2017-10-06 23:18:17 +0100331
332 return err;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800333}
334
Chris Wilsone27ab732017-06-15 13:38:49 +0100335static void __start_cpu_write(struct drm_i915_gem_object *obj)
336{
Christian Königc0a51fd2018-02-16 13:43:38 +0100337 obj->read_domains = I915_GEM_DOMAIN_CPU;
338 obj->write_domain = I915_GEM_DOMAIN_CPU;
Chris Wilsone27ab732017-06-15 13:38:49 +0100339 if (cpu_write_needs_clflush(obj))
340 obj->cache_dirty = true;
341}
342
Chris Wilson6a2c4232014-11-04 04:51:40 -0800343static void
Chris Wilson2b3c8312016-11-11 14:58:09 +0000344__i915_gem_object_release_shmem(struct drm_i915_gem_object *obj,
Chris Wilsone5facdf2016-12-23 14:57:57 +0000345 struct sg_table *pages,
346 bool needs_clflush)
Chris Wilson6a2c4232014-11-04 04:51:40 -0800347{
Chris Wilsona4f5ea62016-10-28 13:58:35 +0100348 GEM_BUG_ON(obj->mm.madv == __I915_MADV_PURGED);
Chris Wilson6a2c4232014-11-04 04:51:40 -0800349
Chris Wilsona4f5ea62016-10-28 13:58:35 +0100350 if (obj->mm.madv == I915_MADV_DONTNEED)
351 obj->mm.dirty = false;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800352
Chris Wilsone5facdf2016-12-23 14:57:57 +0000353 if (needs_clflush &&
Christian Königc0a51fd2018-02-16 13:43:38 +0100354 (obj->read_domains & I915_GEM_DOMAIN_CPU) == 0 &&
Chris Wilsonb8f55be2017-08-11 12:11:16 +0100355 !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ))
Chris Wilson2b3c8312016-11-11 14:58:09 +0000356 drm_clflush_sg(pages);
Chris Wilson03ac84f2016-10-28 13:58:36 +0100357
Chris Wilsone27ab732017-06-15 13:38:49 +0100358 __start_cpu_write(obj);
Chris Wilson03ac84f2016-10-28 13:58:36 +0100359}
360
361static void
362i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj,
363 struct sg_table *pages)
364{
Chris Wilsone5facdf2016-12-23 14:57:57 +0000365 __i915_gem_object_release_shmem(obj, pages, false);
Chris Wilson03ac84f2016-10-28 13:58:36 +0100366
Chris Wilsona4f5ea62016-10-28 13:58:35 +0100367 if (obj->mm.dirty) {
Al Viro93c76a32015-12-04 23:45:44 -0500368 struct address_space *mapping = obj->base.filp->f_mapping;
Chris Wilson6a2c4232014-11-04 04:51:40 -0800369 char *vaddr = obj->phys_handle->vaddr;
Chris Wilson00731152014-05-21 12:42:56 +0100370 int i;
371
372 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
Chris Wilson6a2c4232014-11-04 04:51:40 -0800373 struct page *page;
374 char *dst;
Chris Wilson00731152014-05-21 12:42:56 +0100375
Chris Wilson6a2c4232014-11-04 04:51:40 -0800376 page = shmem_read_mapping_page(mapping, i);
377 if (IS_ERR(page))
378 continue;
379
380 dst = kmap_atomic(page);
381 drm_clflush_virt_range(vaddr, PAGE_SIZE);
382 memcpy(dst, vaddr, PAGE_SIZE);
383 kunmap_atomic(dst);
384
385 set_page_dirty(page);
Chris Wilsona4f5ea62016-10-28 13:58:35 +0100386 if (obj->mm.madv == I915_MADV_WILLNEED)
Chris Wilson00731152014-05-21 12:42:56 +0100387 mark_page_accessed(page);
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300388 put_page(page);
Chris Wilson00731152014-05-21 12:42:56 +0100389 vaddr += PAGE_SIZE;
390 }
Chris Wilsona4f5ea62016-10-28 13:58:35 +0100391 obj->mm.dirty = false;
Chris Wilson00731152014-05-21 12:42:56 +0100392 }
393
Chris Wilson03ac84f2016-10-28 13:58:36 +0100394 sg_free_table(pages);
395 kfree(pages);
Chris Wilsondbb43512016-12-07 13:34:11 +0000396
397 drm_pci_free(obj->base.dev, obj->phys_handle);
Chris Wilson6a2c4232014-11-04 04:51:40 -0800398}
399
400static void
401i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
402{
Chris Wilsona4f5ea62016-10-28 13:58:35 +0100403 i915_gem_object_unpin_pages(obj);
Chris Wilson6a2c4232014-11-04 04:51:40 -0800404}
405
406static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
407 .get_pages = i915_gem_object_get_pages_phys,
408 .put_pages = i915_gem_object_put_pages_phys,
409 .release = i915_gem_object_release_phys,
410};
411
Chris Wilson581ab1f2017-02-15 16:39:00 +0000412static const struct drm_i915_gem_object_ops i915_gem_object_ops;
413
Chris Wilson35a96112016-08-14 18:44:40 +0100414int i915_gem_object_unbind(struct drm_i915_gem_object *obj)
Chris Wilsonaa653a62016-08-04 07:52:27 +0100415{
416 struct i915_vma *vma;
417 LIST_HEAD(still_in_list);
Chris Wilson02bef8f2016-08-14 18:44:41 +0100418 int ret;
Chris Wilsonaa653a62016-08-04 07:52:27 +0100419
Chris Wilson02bef8f2016-08-14 18:44:41 +0100420 lockdep_assert_held(&obj->base.dev->struct_mutex);
421
422 /* Closed vma are removed from the obj->vma_list - but they may
423 * still have an active binding on the object. To remove those we
424 * must wait for all rendering to complete to the object (as unbinding
425 * must anyway), and retire the requests.
Chris Wilsonaa653a62016-08-04 07:52:27 +0100426 */
Chris Wilson5888fc92017-12-04 13:25:13 +0000427 ret = i915_gem_object_set_to_cpu_domain(obj, false);
Chris Wilson02bef8f2016-08-14 18:44:41 +0100428 if (ret)
429 return ret;
430
Chris Wilsonaa653a62016-08-04 07:52:27 +0100431 while ((vma = list_first_entry_or_null(&obj->vma_list,
432 struct i915_vma,
433 obj_link))) {
434 list_move_tail(&vma->obj_link, &still_in_list);
435 ret = i915_vma_unbind(vma);
436 if (ret)
437 break;
438 }
439 list_splice(&still_in_list, &obj->vma_list);
440
441 return ret;
442}
443
Chris Wilsone95433c2016-10-28 13:58:27 +0100444static long
445i915_gem_object_wait_fence(struct dma_fence *fence,
446 unsigned int flags,
447 long timeout,
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100448 struct intel_rps_client *rps_client)
Chris Wilsone95433c2016-10-28 13:58:27 +0100449{
Chris Wilsone61e0f52018-02-21 09:56:36 +0000450 struct i915_request *rq;
Chris Wilsone95433c2016-10-28 13:58:27 +0100451
452 BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
453
454 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
455 return timeout;
456
457 if (!dma_fence_is_i915(fence))
458 return dma_fence_wait_timeout(fence,
459 flags & I915_WAIT_INTERRUPTIBLE,
460 timeout);
461
462 rq = to_request(fence);
Chris Wilsone61e0f52018-02-21 09:56:36 +0000463 if (i915_request_completed(rq))
Chris Wilsone95433c2016-10-28 13:58:27 +0100464 goto out;
465
Chris Wilsone9af4ea2018-01-18 13:16:09 +0000466 /*
467 * This client is about to stall waiting for the GPU. In many cases
Chris Wilsone95433c2016-10-28 13:58:27 +0100468 * this is undesirable and limits the throughput of the system, as
469 * many clients cannot continue processing user input/output whilst
470 * blocked. RPS autotuning may take tens of milliseconds to respond
471 * to the GPU load and thus incurs additional latency for the client.
472 * We can circumvent that by promoting the GPU frequency to maximum
473 * before we wait. This makes the GPU throttle up much more quickly
474 * (good for benchmarks and user experience, e.g. window animations),
475 * but at a cost of spending more power processing the workload
476 * (bad for battery). Not all clients even want their results
477 * immediately and for them we should just let the GPU select its own
478 * frequency to maximise efficiency. To prevent a single client from
479 * forcing the clocks too high for the whole system, we only allow
480 * each client to waitboost once in a busy period.
481 */
Chris Wilsone61e0f52018-02-21 09:56:36 +0000482 if (rps_client && !i915_request_started(rq)) {
Chris Wilsone95433c2016-10-28 13:58:27 +0100483 if (INTEL_GEN(rq->i915) >= 6)
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100484 gen6_rps_boost(rq, rps_client);
Chris Wilsone95433c2016-10-28 13:58:27 +0100485 }
486
Chris Wilsone61e0f52018-02-21 09:56:36 +0000487 timeout = i915_request_wait(rq, flags, timeout);
Chris Wilsone95433c2016-10-28 13:58:27 +0100488
489out:
Chris Wilsone61e0f52018-02-21 09:56:36 +0000490 if (flags & I915_WAIT_LOCKED && i915_request_completed(rq))
491 i915_request_retire_upto(rq);
Chris Wilsone95433c2016-10-28 13:58:27 +0100492
Chris Wilsone95433c2016-10-28 13:58:27 +0100493 return timeout;
494}
495
496static long
497i915_gem_object_wait_reservation(struct reservation_object *resv,
498 unsigned int flags,
499 long timeout,
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100500 struct intel_rps_client *rps_client)
Chris Wilsone95433c2016-10-28 13:58:27 +0100501{
Chris Wilsone54ca972017-02-17 15:13:04 +0000502 unsigned int seq = __read_seqcount_begin(&resv->seq);
Chris Wilsone95433c2016-10-28 13:58:27 +0100503 struct dma_fence *excl;
Chris Wilsone54ca972017-02-17 15:13:04 +0000504 bool prune_fences = false;
Chris Wilsone95433c2016-10-28 13:58:27 +0100505
506 if (flags & I915_WAIT_ALL) {
507 struct dma_fence **shared;
508 unsigned int count, i;
509 int ret;
510
511 ret = reservation_object_get_fences_rcu(resv,
512 &excl, &count, &shared);
513 if (ret)
514 return ret;
515
516 for (i = 0; i < count; i++) {
517 timeout = i915_gem_object_wait_fence(shared[i],
518 flags, timeout,
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100519 rps_client);
Chris Wilsond892e932017-02-12 21:53:43 +0000520 if (timeout < 0)
Chris Wilsone95433c2016-10-28 13:58:27 +0100521 break;
522
523 dma_fence_put(shared[i]);
524 }
525
526 for (; i < count; i++)
527 dma_fence_put(shared[i]);
528 kfree(shared);
Chris Wilsone54ca972017-02-17 15:13:04 +0000529
Chris Wilsonfa730552018-03-07 17:13:03 +0000530 /*
531 * If both shared fences and an exclusive fence exist,
532 * then by construction the shared fences must be later
533 * than the exclusive fence. If we successfully wait for
534 * all the shared fences, we know that the exclusive fence
535 * must all be signaled. If all the shared fences are
536 * signaled, we can prune the array and recover the
537 * floating references on the fences/requests.
538 */
Chris Wilsone54ca972017-02-17 15:13:04 +0000539 prune_fences = count && timeout >= 0;
Chris Wilsone95433c2016-10-28 13:58:27 +0100540 } else {
541 excl = reservation_object_get_excl_rcu(resv);
542 }
543
Chris Wilsonfa730552018-03-07 17:13:03 +0000544 if (excl && timeout >= 0)
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100545 timeout = i915_gem_object_wait_fence(excl, flags, timeout,
546 rps_client);
Chris Wilsone95433c2016-10-28 13:58:27 +0100547
548 dma_fence_put(excl);
549
Chris Wilsonfa730552018-03-07 17:13:03 +0000550 /*
551 * Opportunistically prune the fences iff we know they have *all* been
Chris Wilson03d1cac2017-03-08 13:26:28 +0000552 * signaled and that the reservation object has not been changed (i.e.
553 * no new fences have been added).
554 */
Chris Wilsone54ca972017-02-17 15:13:04 +0000555 if (prune_fences && !__read_seqcount_retry(&resv->seq, seq)) {
Chris Wilson03d1cac2017-03-08 13:26:28 +0000556 if (reservation_object_trylock(resv)) {
557 if (!__read_seqcount_retry(&resv->seq, seq))
558 reservation_object_add_excl_fence(resv, NULL);
559 reservation_object_unlock(resv);
560 }
Chris Wilsone54ca972017-02-17 15:13:04 +0000561 }
562
Chris Wilsone95433c2016-10-28 13:58:27 +0100563 return timeout;
564}
565
Chris Wilson6b5e90f2016-11-14 20:41:05 +0000566static void __fence_set_priority(struct dma_fence *fence, int prio)
567{
Chris Wilsone61e0f52018-02-21 09:56:36 +0000568 struct i915_request *rq;
Chris Wilson6b5e90f2016-11-14 20:41:05 +0000569 struct intel_engine_cs *engine;
570
Chris Wilsonc218ee02018-01-06 10:56:18 +0000571 if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
Chris Wilson6b5e90f2016-11-14 20:41:05 +0000572 return;
573
574 rq = to_request(fence);
575 engine = rq->engine;
Chris Wilson6b5e90f2016-11-14 20:41:05 +0000576
Chris Wilson47650db2018-03-07 13:42:25 +0000577 rcu_read_lock();
578 if (engine->schedule)
579 engine->schedule(rq, prio);
580 rcu_read_unlock();
Chris Wilson6b5e90f2016-11-14 20:41:05 +0000581}
582
583static void fence_set_priority(struct dma_fence *fence, int prio)
584{
585 /* Recurse once into a fence-array */
586 if (dma_fence_is_array(fence)) {
587 struct dma_fence_array *array = to_dma_fence_array(fence);
588 int i;
589
590 for (i = 0; i < array->num_fences; i++)
591 __fence_set_priority(array->fences[i], prio);
592 } else {
593 __fence_set_priority(fence, prio);
594 }
595}
596
597int
598i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
599 unsigned int flags,
600 int prio)
601{
602 struct dma_fence *excl;
603
604 if (flags & I915_WAIT_ALL) {
605 struct dma_fence **shared;
606 unsigned int count, i;
607 int ret;
608
609 ret = reservation_object_get_fences_rcu(obj->resv,
610 &excl, &count, &shared);
611 if (ret)
612 return ret;
613
614 for (i = 0; i < count; i++) {
615 fence_set_priority(shared[i], prio);
616 dma_fence_put(shared[i]);
617 }
618
619 kfree(shared);
620 } else {
621 excl = reservation_object_get_excl_rcu(obj->resv);
622 }
623
624 if (excl) {
625 fence_set_priority(excl, prio);
626 dma_fence_put(excl);
627 }
628 return 0;
629}
630
Chris Wilson00e60f22016-08-04 16:32:40 +0100631/**
Chris Wilsone95433c2016-10-28 13:58:27 +0100632 * Waits for rendering to the object to be completed
Chris Wilson00e60f22016-08-04 16:32:40 +0100633 * @obj: i915 gem object
Chris Wilsone95433c2016-10-28 13:58:27 +0100634 * @flags: how to wait (under a lock, for all rendering or just for writes etc)
635 * @timeout: how long to wait
Chris Wilsona0a8b1c2017-11-09 14:06:44 +0000636 * @rps_client: client (user process) to charge for any waitboosting
Chris Wilson00e60f22016-08-04 16:32:40 +0100637 */
638int
Chris Wilsone95433c2016-10-28 13:58:27 +0100639i915_gem_object_wait(struct drm_i915_gem_object *obj,
640 unsigned int flags,
641 long timeout,
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100642 struct intel_rps_client *rps_client)
Chris Wilson00e60f22016-08-04 16:32:40 +0100643{
Chris Wilsone95433c2016-10-28 13:58:27 +0100644 might_sleep();
645#if IS_ENABLED(CONFIG_LOCKDEP)
646 GEM_BUG_ON(debug_locks &&
647 !!lockdep_is_held(&obj->base.dev->struct_mutex) !=
648 !!(flags & I915_WAIT_LOCKED));
649#endif
650 GEM_BUG_ON(timeout < 0);
Chris Wilson00e60f22016-08-04 16:32:40 +0100651
Chris Wilsond07f0e52016-10-28 13:58:44 +0100652 timeout = i915_gem_object_wait_reservation(obj->resv,
653 flags, timeout,
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100654 rps_client);
Chris Wilsone95433c2016-10-28 13:58:27 +0100655 return timeout < 0 ? timeout : 0;
Chris Wilson00e60f22016-08-04 16:32:40 +0100656}
657
658static struct intel_rps_client *to_rps_client(struct drm_file *file)
659{
660 struct drm_i915_file_private *fpriv = file->driver_priv;
661
Sagar Arun Kamble562d9ba2017-10-10 22:30:06 +0100662 return &fpriv->rps_client;
Chris Wilson00e60f22016-08-04 16:32:40 +0100663}
664
Chris Wilson00731152014-05-21 12:42:56 +0100665static int
666i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
667 struct drm_i915_gem_pwrite *args,
Chris Wilson03ac84f2016-10-28 13:58:36 +0100668 struct drm_file *file)
Chris Wilson00731152014-05-21 12:42:56 +0100669{
Chris Wilson00731152014-05-21 12:42:56 +0100670 void *vaddr = obj->phys_handle->vaddr + args->offset;
Gustavo Padovan3ed605b2016-04-26 12:32:27 -0300671 char __user *user_data = u64_to_user_ptr(args->data_ptr);
Chris Wilson6a2c4232014-11-04 04:51:40 -0800672
673 /* We manually control the domain here and pretend that it
674 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
675 */
Rodrigo Vivi77a0d1c2015-06-18 11:43:24 -0700676 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
Chris Wilson10466d22017-01-06 15:22:38 +0000677 if (copy_from_user(vaddr, user_data, args->size))
678 return -EFAULT;
Chris Wilson00731152014-05-21 12:42:56 +0100679
Chris Wilson6a2c4232014-11-04 04:51:40 -0800680 drm_clflush_virt_range(vaddr, args->size);
Chris Wilson10466d22017-01-06 15:22:38 +0000681 i915_gem_chipset_flush(to_i915(obj->base.dev));
Paulo Zanoni063e4e62015-02-13 17:23:45 -0200682
Chris Wilsond59b21e2017-02-22 11:40:49 +0000683 intel_fb_obj_flush(obj, ORIGIN_CPU);
Chris Wilson10466d22017-01-06 15:22:38 +0000684 return 0;
Chris Wilson00731152014-05-21 12:42:56 +0100685}
686
Tvrtko Ursulin187685c2016-12-01 14:16:36 +0000687void *i915_gem_object_alloc(struct drm_i915_private *dev_priv)
Chris Wilson42dcedd2012-11-15 11:32:30 +0000688{
Chris Wilsonefab6d82015-04-07 16:20:57 +0100689 return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
Chris Wilson42dcedd2012-11-15 11:32:30 +0000690}
691
692void i915_gem_object_free(struct drm_i915_gem_object *obj)
693{
Chris Wilsonfac5e232016-07-04 11:34:36 +0100694 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
Chris Wilsonefab6d82015-04-07 16:20:57 +0100695 kmem_cache_free(dev_priv->objects, obj);
Chris Wilson42dcedd2012-11-15 11:32:30 +0000696}
697
Dave Airlieff72145b2011-02-07 12:16:14 +1000698static int
699i915_gem_create(struct drm_file *file,
Tvrtko Ursulin12d79d72016-12-01 14:16:37 +0000700 struct drm_i915_private *dev_priv,
Dave Airlieff72145b2011-02-07 12:16:14 +1000701 uint64_t size,
702 uint32_t *handle_p)
Eric Anholt673a3942008-07-30 12:06:12 -0700703{
Chris Wilson05394f32010-11-08 19:18:58 +0000704 struct drm_i915_gem_object *obj;
Pekka Paalanena1a2d1d2009-08-23 12:40:55 +0300705 int ret;
706 u32 handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700707
Dave Airlieff72145b2011-02-07 12:16:14 +1000708 size = roundup(size, PAGE_SIZE);
Chris Wilson8ffc0242011-09-14 14:14:28 +0200709 if (size == 0)
710 return -EINVAL;
Eric Anholt673a3942008-07-30 12:06:12 -0700711
712 /* Allocate the new object */
Tvrtko Ursulin12d79d72016-12-01 14:16:37 +0000713 obj = i915_gem_object_create(dev_priv, size);
Chris Wilsonfe3db792016-04-25 13:32:13 +0100714 if (IS_ERR(obj))
715 return PTR_ERR(obj);
Eric Anholt673a3942008-07-30 12:06:12 -0700716
Chris Wilson05394f32010-11-08 19:18:58 +0000717 ret = drm_gem_handle_create(file, &obj->base, &handle);
Chris Wilson202f2fe2010-10-14 13:20:40 +0100718 /* drop reference from allocate - handle holds it now */
Chris Wilsonf0cd5182016-10-28 13:58:43 +0100719 i915_gem_object_put(obj);
Daniel Vetterd861e332013-07-24 23:25:03 +0200720 if (ret)
721 return ret;
Chris Wilson202f2fe2010-10-14 13:20:40 +0100722
Dave Airlieff72145b2011-02-07 12:16:14 +1000723 *handle_p = handle;
Eric Anholt673a3942008-07-30 12:06:12 -0700724 return 0;
725}
726
Dave Airlieff72145b2011-02-07 12:16:14 +1000727int
728i915_gem_dumb_create(struct drm_file *file,
729 struct drm_device *dev,
730 struct drm_mode_create_dumb *args)
731{
732 /* have to work out size/pitch and return them */
Paulo Zanonide45eaf2013-10-18 18:48:24 -0300733 args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
Dave Airlieff72145b2011-02-07 12:16:14 +1000734 args->size = args->pitch * args->height;
Tvrtko Ursulin12d79d72016-12-01 14:16:37 +0000735 return i915_gem_create(file, to_i915(dev),
Dave Airlieda6b51d2014-12-24 13:11:17 +1000736 args->size, &args->handle);
Dave Airlieff72145b2011-02-07 12:16:14 +1000737}
738
Chris Wilsone27ab732017-06-15 13:38:49 +0100739static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)
740{
741 return !(obj->cache_level == I915_CACHE_NONE ||
742 obj->cache_level == I915_CACHE_WT);
743}
744
Dave Airlieff72145b2011-02-07 12:16:14 +1000745/**
746 * Creates a new mm object and returns a handle to it.
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +0100747 * @dev: drm device pointer
748 * @data: ioctl data blob
749 * @file: drm file pointer
Dave Airlieff72145b2011-02-07 12:16:14 +1000750 */
751int
752i915_gem_create_ioctl(struct drm_device *dev, void *data,
753 struct drm_file *file)
754{
Tvrtko Ursulin12d79d72016-12-01 14:16:37 +0000755 struct drm_i915_private *dev_priv = to_i915(dev);
Dave Airlieff72145b2011-02-07 12:16:14 +1000756 struct drm_i915_gem_create *args = data;
Daniel Vetter63ed2cb2012-04-23 16:50:50 +0200757
Tvrtko Ursulin12d79d72016-12-01 14:16:37 +0000758 i915_gem_flush_free_objects(dev_priv);
Chris Wilsonfbbd37b2016-10-28 13:58:42 +0100759
Tvrtko Ursulin12d79d72016-12-01 14:16:37 +0000760 return i915_gem_create(file, dev_priv,
Dave Airlieda6b51d2014-12-24 13:11:17 +1000761 args->size, &args->handle);
Dave Airlieff72145b2011-02-07 12:16:14 +1000762}
763
Chris Wilsonef749212017-04-12 12:01:10 +0100764static inline enum fb_op_origin
765fb_write_origin(struct drm_i915_gem_object *obj, unsigned int domain)
766{
767 return (domain == I915_GEM_DOMAIN_GTT ?
768 obj->frontbuffer_ggtt_origin : ORIGIN_CPU);
769}
770
Chris Wilson71253972017-12-06 12:49:14 +0000771void i915_gem_flush_ggtt_writes(struct drm_i915_private *dev_priv)
Chris Wilsonef749212017-04-12 12:01:10 +0100772{
Chris Wilson71253972017-12-06 12:49:14 +0000773 /*
774 * No actual flushing is required for the GTT write domain for reads
775 * from the GTT domain. Writes to it "immediately" go to main memory
776 * as far as we know, so there's no chipset flush. It also doesn't
777 * land in the GPU render cache.
Chris Wilsonef749212017-04-12 12:01:10 +0100778 *
779 * However, we do have to enforce the order so that all writes through
780 * the GTT land before any writes to the device, such as updates to
781 * the GATT itself.
782 *
783 * We also have to wait a bit for the writes to land from the GTT.
784 * An uncached read (i.e. mmio) seems to be ideal for the round-trip
785 * timing. This issue has only been observed when switching quickly
786 * between GTT writes and CPU reads from inside the kernel on recent hw,
787 * and it appears to only affect discrete GTT blocks (i.e. on LLC
Chris Wilson71253972017-12-06 12:49:14 +0000788 * system agents we cannot reproduce this behaviour, until Cannonlake
789 * that was!).
Chris Wilsonef749212017-04-12 12:01:10 +0100790 */
Chris Wilson71253972017-12-06 12:49:14 +0000791
Chris Wilsonef749212017-04-12 12:01:10 +0100792 wmb();
793
Chris Wilson71253972017-12-06 12:49:14 +0000794 intel_runtime_pm_get(dev_priv);
795 spin_lock_irq(&dev_priv->uncore.lock);
796
797 POSTING_READ_FW(RING_HEAD(RENDER_RING_BASE));
798
799 spin_unlock_irq(&dev_priv->uncore.lock);
800 intel_runtime_pm_put(dev_priv);
801}
802
803static void
804flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
805{
806 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
807 struct i915_vma *vma;
808
Christian Königc0a51fd2018-02-16 13:43:38 +0100809 if (!(obj->write_domain & flush_domains))
Chris Wilson71253972017-12-06 12:49:14 +0000810 return;
811
Christian Königc0a51fd2018-02-16 13:43:38 +0100812 switch (obj->write_domain) {
Chris Wilsonef749212017-04-12 12:01:10 +0100813 case I915_GEM_DOMAIN_GTT:
Chris Wilson71253972017-12-06 12:49:14 +0000814 i915_gem_flush_ggtt_writes(dev_priv);
Chris Wilsonef749212017-04-12 12:01:10 +0100815
816 intel_fb_obj_flush(obj,
817 fb_write_origin(obj, I915_GEM_DOMAIN_GTT));
Chris Wilson71253972017-12-06 12:49:14 +0000818
Chris Wilsone2189dd2017-12-07 21:14:07 +0000819 for_each_ggtt_vma(vma, obj) {
Chris Wilson71253972017-12-06 12:49:14 +0000820 if (vma->iomap)
821 continue;
822
823 i915_vma_unset_ggtt_write(vma);
824 }
Chris Wilsonef749212017-04-12 12:01:10 +0100825 break;
826
827 case I915_GEM_DOMAIN_CPU:
828 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
829 break;
Chris Wilsone27ab732017-06-15 13:38:49 +0100830
831 case I915_GEM_DOMAIN_RENDER:
832 if (gpu_write_needs_clflush(obj))
833 obj->cache_dirty = true;
834 break;
Chris Wilsonef749212017-04-12 12:01:10 +0100835 }
836
Christian Königc0a51fd2018-02-16 13:43:38 +0100837 obj->write_domain = 0;
Chris Wilsonef749212017-04-12 12:01:10 +0100838}
839
Daniel Vetter8c599672011-12-14 13:57:31 +0100840static inline int
Daniel Vetter8461d222011-12-14 13:57:32 +0100841__copy_to_user_swizzled(char __user *cpu_vaddr,
842 const char *gpu_vaddr, int gpu_offset,
843 int length)
844{
845 int ret, cpu_offset = 0;
846
847 while (length > 0) {
848 int cacheline_end = ALIGN(gpu_offset + 1, 64);
849 int this_length = min(cacheline_end - gpu_offset, length);
850 int swizzled_gpu_offset = gpu_offset ^ 64;
851
852 ret = __copy_to_user(cpu_vaddr + cpu_offset,
853 gpu_vaddr + swizzled_gpu_offset,
854 this_length);
855 if (ret)
856 return ret + length;
857
858 cpu_offset += this_length;
859 gpu_offset += this_length;
860 length -= this_length;
861 }
862
863 return 0;
864}
865
866static inline int
Ben Widawsky4f0c7cf2012-04-16 14:07:47 -0700867__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
868 const char __user *cpu_vaddr,
Daniel Vetter8c599672011-12-14 13:57:31 +0100869 int length)
870{
871 int ret, cpu_offset = 0;
872
873 while (length > 0) {
874 int cacheline_end = ALIGN(gpu_offset + 1, 64);
875 int this_length = min(cacheline_end - gpu_offset, length);
876 int swizzled_gpu_offset = gpu_offset ^ 64;
877
878 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
879 cpu_vaddr + cpu_offset,
880 this_length);
881 if (ret)
882 return ret + length;
883
884 cpu_offset += this_length;
885 gpu_offset += this_length;
886 length -= this_length;
887 }
888
889 return 0;
890}
891
Brad Volkin4c914c02014-02-18 10:15:45 -0800892/*
893 * Pins the specified object's pages and synchronizes the object with
894 * GPU accesses. Sets needs_clflush to non-zero if the caller should
895 * flush the object from the CPU cache.
896 */
897int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
Chris Wilson43394c72016-08-18 17:16:47 +0100898 unsigned int *needs_clflush)
Brad Volkin4c914c02014-02-18 10:15:45 -0800899{
900 int ret;
901
Chris Wilsone95433c2016-10-28 13:58:27 +0100902 lockdep_assert_held(&obj->base.dev->struct_mutex);
Brad Volkin4c914c02014-02-18 10:15:45 -0800903
Chris Wilsone95433c2016-10-28 13:58:27 +0100904 *needs_clflush = 0;
Chris Wilson43394c72016-08-18 17:16:47 +0100905 if (!i915_gem_object_has_struct_page(obj))
906 return -ENODEV;
Brad Volkin4c914c02014-02-18 10:15:45 -0800907
Chris Wilsone95433c2016-10-28 13:58:27 +0100908 ret = i915_gem_object_wait(obj,
909 I915_WAIT_INTERRUPTIBLE |
910 I915_WAIT_LOCKED,
911 MAX_SCHEDULE_TIMEOUT,
912 NULL);
Chris Wilsonc13d87e2016-07-20 09:21:15 +0100913 if (ret)
914 return ret;
915
Chris Wilsona4f5ea62016-10-28 13:58:35 +0100916 ret = i915_gem_object_pin_pages(obj);
Chris Wilson97649512016-08-18 17:16:50 +0100917 if (ret)
918 return ret;
919
Chris Wilsonb8f55be2017-08-11 12:11:16 +0100920 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
921 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
Chris Wilson7f5f95d2017-03-10 00:09:42 +0000922 ret = i915_gem_object_set_to_cpu_domain(obj, false);
923 if (ret)
924 goto err_unpin;
925 else
926 goto out;
927 }
928
Chris Wilsonef749212017-04-12 12:01:10 +0100929 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
Chris Wilsona314d5c2016-08-18 17:16:48 +0100930
Chris Wilson43394c72016-08-18 17:16:47 +0100931 /* If we're not in the cpu read domain, set ourself into the gtt
932 * read domain and manually flush cachelines (if required). This
933 * optimizes for the case when the gpu will dirty the data
934 * anyway again before the next pread happens.
935 */
Chris Wilsone27ab732017-06-15 13:38:49 +0100936 if (!obj->cache_dirty &&
Christian Königc0a51fd2018-02-16 13:43:38 +0100937 !(obj->read_domains & I915_GEM_DOMAIN_CPU))
Chris Wilson7f5f95d2017-03-10 00:09:42 +0000938 *needs_clflush = CLFLUSH_BEFORE;
Brad Volkin4c914c02014-02-18 10:15:45 -0800939
Chris Wilson7f5f95d2017-03-10 00:09:42 +0000940out:
Chris Wilson97649512016-08-18 17:16:50 +0100941 /* return with the pages pinned */
Chris Wilson43394c72016-08-18 17:16:47 +0100942 return 0;
Chris Wilson97649512016-08-18 17:16:50 +0100943
944err_unpin:
945 i915_gem_object_unpin_pages(obj);
946 return ret;
Chris Wilson43394c72016-08-18 17:16:47 +0100947}
948
949int i915_gem_obj_prepare_shmem_write(struct drm_i915_gem_object *obj,
950 unsigned int *needs_clflush)
951{
952 int ret;
953
Chris Wilsone95433c2016-10-28 13:58:27 +0100954 lockdep_assert_held(&obj->base.dev->struct_mutex);
955
Chris Wilson43394c72016-08-18 17:16:47 +0100956 *needs_clflush = 0;
957 if (!i915_gem_object_has_struct_page(obj))
958 return -ENODEV;
959
Chris Wilsone95433c2016-10-28 13:58:27 +0100960 ret = i915_gem_object_wait(obj,
961 I915_WAIT_INTERRUPTIBLE |
962 I915_WAIT_LOCKED |
963 I915_WAIT_ALL,
964 MAX_SCHEDULE_TIMEOUT,
965 NULL);
Chris Wilson43394c72016-08-18 17:16:47 +0100966 if (ret)
967 return ret;
968
Chris Wilsona4f5ea62016-10-28 13:58:35 +0100969 ret = i915_gem_object_pin_pages(obj);
Chris Wilson97649512016-08-18 17:16:50 +0100970 if (ret)
971 return ret;
972
Chris Wilsonb8f55be2017-08-11 12:11:16 +0100973 if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
974 !static_cpu_has(X86_FEATURE_CLFLUSH)) {
Chris Wilson7f5f95d2017-03-10 00:09:42 +0000975 ret = i915_gem_object_set_to_cpu_domain(obj, true);
976 if (ret)
977 goto err_unpin;
978 else
979 goto out;
980 }
981
Chris Wilsonef749212017-04-12 12:01:10 +0100982 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
Chris Wilsona314d5c2016-08-18 17:16:48 +0100983
Chris Wilson43394c72016-08-18 17:16:47 +0100984 /* If we're not in the cpu write domain, set ourself into the
985 * gtt write domain and manually flush cachelines (as required).
986 * This optimizes for the case when the gpu will use the data
987 * right away and we therefore have to clflush anyway.
988 */
Chris Wilsone27ab732017-06-15 13:38:49 +0100989 if (!obj->cache_dirty) {
Chris Wilson7f5f95d2017-03-10 00:09:42 +0000990 *needs_clflush |= CLFLUSH_AFTER;
Chris Wilson43394c72016-08-18 17:16:47 +0100991
Chris Wilsone27ab732017-06-15 13:38:49 +0100992 /*
993 * Same trick applies to invalidate partially written
994 * cachelines read before writing.
995 */
Christian Königc0a51fd2018-02-16 13:43:38 +0100996 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))
Chris Wilsone27ab732017-06-15 13:38:49 +0100997 *needs_clflush |= CLFLUSH_BEFORE;
998 }
Chris Wilson43394c72016-08-18 17:16:47 +0100999
Chris Wilson7f5f95d2017-03-10 00:09:42 +00001000out:
Chris Wilson43394c72016-08-18 17:16:47 +01001001 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
Chris Wilsona4f5ea62016-10-28 13:58:35 +01001002 obj->mm.dirty = true;
Chris Wilson97649512016-08-18 17:16:50 +01001003 /* return with the pages pinned */
Chris Wilson43394c72016-08-18 17:16:47 +01001004 return 0;
Chris Wilson97649512016-08-18 17:16:50 +01001005
1006err_unpin:
1007 i915_gem_object_unpin_pages(obj);
1008 return ret;
Brad Volkin4c914c02014-02-18 10:15:45 -08001009}
1010
Daniel Vetter23c18c72012-03-25 19:47:42 +02001011static void
1012shmem_clflush_swizzled_range(char *addr, unsigned long length,
1013 bool swizzled)
1014{
Daniel Vettere7e58eb2012-03-25 19:47:43 +02001015 if (unlikely(swizzled)) {
Daniel Vetter23c18c72012-03-25 19:47:42 +02001016 unsigned long start = (unsigned long) addr;
1017 unsigned long end = (unsigned long) addr + length;
1018
1019 /* For swizzling simply ensure that we always flush both
1020 * channels. Lame, but simple and it works. Swizzled
1021 * pwrite/pread is far from a hotpath - current userspace
1022 * doesn't use it at all. */
1023 start = round_down(start, 128);
1024 end = round_up(end, 128);
1025
1026 drm_clflush_virt_range((void *)start, end - start);
1027 } else {
1028 drm_clflush_virt_range(addr, length);
1029 }
1030
1031}
1032
Daniel Vetterd174bd62012-03-25 19:47:40 +02001033/* Only difference to the fast-path function is that this can handle bit17
1034 * and uses non-atomic copy and kmap functions. */
1035static int
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001036shmem_pread_slow(struct page *page, int offset, int length,
Daniel Vetterd174bd62012-03-25 19:47:40 +02001037 char __user *user_data,
1038 bool page_do_bit17_swizzling, bool needs_clflush)
1039{
1040 char *vaddr;
1041 int ret;
1042
1043 vaddr = kmap(page);
1044 if (needs_clflush)
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001045 shmem_clflush_swizzled_range(vaddr + offset, length,
Daniel Vetter23c18c72012-03-25 19:47:42 +02001046 page_do_bit17_swizzling);
Daniel Vetterd174bd62012-03-25 19:47:40 +02001047
1048 if (page_do_bit17_swizzling)
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001049 ret = __copy_to_user_swizzled(user_data, vaddr, offset, length);
Daniel Vetterd174bd62012-03-25 19:47:40 +02001050 else
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001051 ret = __copy_to_user(user_data, vaddr + offset, length);
Daniel Vetterd174bd62012-03-25 19:47:40 +02001052 kunmap(page);
1053
Chris Wilsonf60d7f02012-09-04 21:02:56 +01001054 return ret ? - EFAULT : 0;
Daniel Vetterd174bd62012-03-25 19:47:40 +02001055}
1056
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001057static int
1058shmem_pread(struct page *page, int offset, int length, char __user *user_data,
1059 bool page_do_bit17_swizzling, bool needs_clflush)
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301060{
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001061 int ret;
1062
1063 ret = -ENODEV;
1064 if (!page_do_bit17_swizzling) {
1065 char *vaddr = kmap_atomic(page);
1066
1067 if (needs_clflush)
1068 drm_clflush_virt_range(vaddr + offset, length);
1069 ret = __copy_to_user_inatomic(user_data, vaddr + offset, length);
1070 kunmap_atomic(vaddr);
1071 }
1072 if (ret == 0)
1073 return 0;
1074
1075 return shmem_pread_slow(page, offset, length, user_data,
1076 page_do_bit17_swizzling, needs_clflush);
1077}
1078
1079static int
1080i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
1081 struct drm_i915_gem_pread *args)
1082{
1083 char __user *user_data;
1084 u64 remain;
1085 unsigned int obj_do_bit17_swizzling;
1086 unsigned int needs_clflush;
1087 unsigned int idx, offset;
1088 int ret;
1089
1090 obj_do_bit17_swizzling = 0;
1091 if (i915_gem_object_needs_bit17_swizzle(obj))
1092 obj_do_bit17_swizzling = BIT(17);
1093
1094 ret = mutex_lock_interruptible(&obj->base.dev->struct_mutex);
1095 if (ret)
1096 return ret;
1097
1098 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
1099 mutex_unlock(&obj->base.dev->struct_mutex);
1100 if (ret)
1101 return ret;
1102
1103 remain = args->size;
1104 user_data = u64_to_user_ptr(args->data_ptr);
1105 offset = offset_in_page(args->offset);
1106 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
1107 struct page *page = i915_gem_object_get_page(obj, idx);
1108 int length;
1109
1110 length = remain;
1111 if (offset + length > PAGE_SIZE)
1112 length = PAGE_SIZE - offset;
1113
1114 ret = shmem_pread(page, offset, length, user_data,
1115 page_to_phys(page) & obj_do_bit17_swizzling,
1116 needs_clflush);
1117 if (ret)
1118 break;
1119
1120 remain -= length;
1121 user_data += length;
1122 offset = 0;
1123 }
1124
1125 i915_gem_obj_finish_shmem_access(obj);
1126 return ret;
1127}
1128
1129static inline bool
1130gtt_user_read(struct io_mapping *mapping,
1131 loff_t base, int offset,
1132 char __user *user_data, int length)
1133{
Ville Syrjäläafe722b2017-09-01 20:12:52 +03001134 void __iomem *vaddr;
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001135 unsigned long unwritten;
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301136
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301137 /* We can use the cpu mem copy function because this is X86. */
Ville Syrjäläafe722b2017-09-01 20:12:52 +03001138 vaddr = io_mapping_map_atomic_wc(mapping, base);
1139 unwritten = __copy_to_user_inatomic(user_data,
1140 (void __force *)vaddr + offset,
1141 length);
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001142 io_mapping_unmap_atomic(vaddr);
1143 if (unwritten) {
Ville Syrjäläafe722b2017-09-01 20:12:52 +03001144 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
1145 unwritten = copy_to_user(user_data,
1146 (void __force *)vaddr + offset,
1147 length);
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001148 io_mapping_unmap(vaddr);
1149 }
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301150 return unwritten;
1151}
1152
1153static int
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001154i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
1155 const struct drm_i915_gem_pread *args)
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301156{
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001157 struct drm_i915_private *i915 = to_i915(obj->base.dev);
1158 struct i915_ggtt *ggtt = &i915->ggtt;
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301159 struct drm_mm_node node;
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001160 struct i915_vma *vma;
1161 void __user *user_data;
1162 u64 remain, offset;
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301163 int ret;
1164
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001165 ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1166 if (ret)
1167 return ret;
1168
1169 intel_runtime_pm_get(i915);
1170 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
Chris Wilsona3259ca2017-10-09 09:44:00 +01001171 PIN_MAPPABLE |
1172 PIN_NONFAULT |
1173 PIN_NONBLOCK);
Chris Wilson18034582016-08-18 17:16:45 +01001174 if (!IS_ERR(vma)) {
1175 node.start = i915_ggtt_offset(vma);
1176 node.allocated = false;
Chris Wilson49ef5292016-08-18 17:17:00 +01001177 ret = i915_vma_put_fence(vma);
Chris Wilson18034582016-08-18 17:16:45 +01001178 if (ret) {
1179 i915_vma_unpin(vma);
1180 vma = ERR_PTR(ret);
1181 }
1182 }
Chris Wilson058d88c2016-08-15 10:49:06 +01001183 if (IS_ERR(vma)) {
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001184 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301185 if (ret)
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001186 goto out_unlock;
1187 GEM_BUG_ON(!node.allocated);
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301188 }
1189
1190 ret = i915_gem_object_set_to_gtt_domain(obj, false);
1191 if (ret)
1192 goto out_unpin;
1193
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001194 mutex_unlock(&i915->drm.struct_mutex);
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301195
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001196 user_data = u64_to_user_ptr(args->data_ptr);
1197 remain = args->size;
1198 offset = args->offset;
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301199
1200 while (remain > 0) {
1201 /* Operation in this page
1202 *
1203 * page_base = page offset within aperture
1204 * page_offset = offset within page
1205 * page_length = bytes to copy for this page
1206 */
1207 u32 page_base = node.start;
1208 unsigned page_offset = offset_in_page(offset);
1209 unsigned page_length = PAGE_SIZE - page_offset;
1210 page_length = remain < page_length ? remain : page_length;
1211 if (node.allocated) {
1212 wmb();
1213 ggtt->base.insert_page(&ggtt->base,
1214 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001215 node.start, I915_CACHE_NONE, 0);
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301216 wmb();
1217 } else {
1218 page_base += offset & PAGE_MASK;
1219 }
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001220
Matthew Auld73ebd502017-12-11 15:18:20 +00001221 if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001222 user_data, page_length)) {
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301223 ret = -EFAULT;
1224 break;
1225 }
1226
1227 remain -= page_length;
1228 user_data += page_length;
1229 offset += page_length;
1230 }
1231
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001232 mutex_lock(&i915->drm.struct_mutex);
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301233out_unpin:
1234 if (node.allocated) {
1235 wmb();
1236 ggtt->base.clear_range(&ggtt->base,
Michał Winiarski4fb84d92016-10-13 14:02:40 +02001237 node.start, node.size);
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301238 remove_mappable_node(&node);
1239 } else {
Chris Wilson058d88c2016-08-15 10:49:06 +01001240 i915_vma_unpin(vma);
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301241 }
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001242out_unlock:
1243 intel_runtime_pm_put(i915);
1244 mutex_unlock(&i915->drm.struct_mutex);
Chris Wilsonf60d7f02012-09-04 21:02:56 +01001245
Eric Anholteb014592009-03-10 11:44:52 -07001246 return ret;
1247}
1248
Eric Anholt673a3942008-07-30 12:06:12 -07001249/**
1250 * Reads data from the object referenced by handle.
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +01001251 * @dev: drm device pointer
1252 * @data: ioctl data blob
1253 * @file: drm file pointer
Eric Anholt673a3942008-07-30 12:06:12 -07001254 *
1255 * On error, the contents of *data are undefined.
1256 */
1257int
1258i915_gem_pread_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001259 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001260{
1261 struct drm_i915_gem_pread *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00001262 struct drm_i915_gem_object *obj;
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001263 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07001264
Chris Wilson51311d02010-11-17 09:10:42 +00001265 if (args->size == 0)
1266 return 0;
1267
1268 if (!access_ok(VERIFY_WRITE,
Gustavo Padovan3ed605b2016-04-26 12:32:27 -03001269 u64_to_user_ptr(args->data_ptr),
Chris Wilson51311d02010-11-17 09:10:42 +00001270 args->size))
1271 return -EFAULT;
1272
Chris Wilson03ac0642016-07-20 13:31:51 +01001273 obj = i915_gem_object_lookup(file, args->handle);
Chris Wilson258a5ed2016-08-05 10:14:16 +01001274 if (!obj)
1275 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07001276
Chris Wilson7dcd2492010-09-26 20:21:44 +01001277 /* Bounds check source. */
Matthew Auld966d5bf2016-12-13 20:32:22 +00001278 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
Chris Wilsonce9d4192010-09-26 20:50:05 +01001279 ret = -EINVAL;
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001280 goto out;
Chris Wilsonce9d4192010-09-26 20:50:05 +01001281 }
1282
Chris Wilsondb53a302011-02-03 11:57:46 +00001283 trace_i915_gem_object_pread(obj, args->offset, args->size);
1284
Chris Wilsone95433c2016-10-28 13:58:27 +01001285 ret = i915_gem_object_wait(obj,
1286 I915_WAIT_INTERRUPTIBLE,
1287 MAX_SCHEDULE_TIMEOUT,
1288 to_rps_client(file));
Chris Wilson258a5ed2016-08-05 10:14:16 +01001289 if (ret)
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001290 goto out;
Chris Wilson258a5ed2016-08-05 10:14:16 +01001291
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001292 ret = i915_gem_object_pin_pages(obj);
Chris Wilson258a5ed2016-08-05 10:14:16 +01001293 if (ret)
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001294 goto out;
Chris Wilson258a5ed2016-08-05 10:14:16 +01001295
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001296 ret = i915_gem_shmem_pread(obj, args);
Chris Wilson9c870d02016-10-24 13:42:15 +01001297 if (ret == -EFAULT || ret == -ENODEV)
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001298 ret = i915_gem_gtt_pread(obj, args);
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301299
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001300 i915_gem_object_unpin_pages(obj);
1301out:
Chris Wilsonf0cd5182016-10-28 13:58:43 +01001302 i915_gem_object_put(obj);
Eric Anholteb014592009-03-10 11:44:52 -07001303 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07001304}
1305
Keith Packard0839ccb2008-10-30 19:38:48 -07001306/* This is the fast write path which cannot handle
1307 * page faults in the source data
Linus Torvalds9b7530cc2008-10-20 14:16:43 -07001308 */
Linus Torvalds9b7530cc2008-10-20 14:16:43 -07001309
Chris Wilsonfe115622016-10-28 13:58:40 +01001310static inline bool
1311ggtt_write(struct io_mapping *mapping,
1312 loff_t base, int offset,
1313 char __user *user_data, int length)
Keith Packard0839ccb2008-10-30 19:38:48 -07001314{
Ville Syrjäläafe722b2017-09-01 20:12:52 +03001315 void __iomem *vaddr;
Keith Packard0839ccb2008-10-30 19:38:48 -07001316 unsigned long unwritten;
1317
Ben Widawsky4f0c7cf2012-04-16 14:07:47 -07001318 /* We can use the cpu mem copy function because this is X86. */
Ville Syrjäläafe722b2017-09-01 20:12:52 +03001319 vaddr = io_mapping_map_atomic_wc(mapping, base);
1320 unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
Keith Packard0839ccb2008-10-30 19:38:48 -07001321 user_data, length);
Chris Wilsonfe115622016-10-28 13:58:40 +01001322 io_mapping_unmap_atomic(vaddr);
1323 if (unwritten) {
Ville Syrjäläafe722b2017-09-01 20:12:52 +03001324 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
1325 unwritten = copy_from_user((void __force *)vaddr + offset,
1326 user_data, length);
Chris Wilsonfe115622016-10-28 13:58:40 +01001327 io_mapping_unmap(vaddr);
1328 }
Keith Packard0839ccb2008-10-30 19:38:48 -07001329
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001330 return unwritten;
1331}
1332
Eric Anholt3de09aa2009-03-09 09:42:23 -07001333/**
1334 * This is the fast pwrite path, where we copy the data directly from the
1335 * user into the GTT, uncached.
Chris Wilsonfe115622016-10-28 13:58:40 +01001336 * @obj: i915 GEM object
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +01001337 * @args: pwrite arguments structure
Eric Anholt3de09aa2009-03-09 09:42:23 -07001338 */
Eric Anholt673a3942008-07-30 12:06:12 -07001339static int
Chris Wilsonfe115622016-10-28 13:58:40 +01001340i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
1341 const struct drm_i915_gem_pwrite *args)
Eric Anholt673a3942008-07-30 12:06:12 -07001342{
Chris Wilsonfe115622016-10-28 13:58:40 +01001343 struct drm_i915_private *i915 = to_i915(obj->base.dev);
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +05301344 struct i915_ggtt *ggtt = &i915->ggtt;
1345 struct drm_mm_node node;
Chris Wilsonfe115622016-10-28 13:58:40 +01001346 struct i915_vma *vma;
1347 u64 remain, offset;
1348 void __user *user_data;
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +05301349 int ret;
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301350
Chris Wilsonfe115622016-10-28 13:58:40 +01001351 ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
1352 if (ret)
1353 return ret;
Daniel Vetter935aaa62012-03-25 19:47:35 +02001354
Chris Wilson8bd81812017-10-19 07:37:33 +01001355 if (i915_gem_object_has_struct_page(obj)) {
1356 /*
1357 * Avoid waking the device up if we can fallback, as
1358 * waking/resuming is very slow (worst-case 10-100 ms
1359 * depending on PCI sleeps and our own resume time).
1360 * This easily dwarfs any performance advantage from
1361 * using the cache bypass of indirect GGTT access.
1362 */
1363 if (!intel_runtime_pm_get_if_in_use(i915)) {
1364 ret = -EFAULT;
1365 goto out_unlock;
1366 }
1367 } else {
1368 /* No backing pages, no fallback, we must force GGTT access */
1369 intel_runtime_pm_get(i915);
1370 }
1371
Chris Wilson058d88c2016-08-15 10:49:06 +01001372 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
Chris Wilsona3259ca2017-10-09 09:44:00 +01001373 PIN_MAPPABLE |
1374 PIN_NONFAULT |
1375 PIN_NONBLOCK);
Chris Wilson18034582016-08-18 17:16:45 +01001376 if (!IS_ERR(vma)) {
1377 node.start = i915_ggtt_offset(vma);
1378 node.allocated = false;
Chris Wilson49ef5292016-08-18 17:17:00 +01001379 ret = i915_vma_put_fence(vma);
Chris Wilson18034582016-08-18 17:16:45 +01001380 if (ret) {
1381 i915_vma_unpin(vma);
1382 vma = ERR_PTR(ret);
1383 }
1384 }
Chris Wilson058d88c2016-08-15 10:49:06 +01001385 if (IS_ERR(vma)) {
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001386 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +05301387 if (ret)
Chris Wilson8bd81812017-10-19 07:37:33 +01001388 goto out_rpm;
Chris Wilsonfe115622016-10-28 13:58:40 +01001389 GEM_BUG_ON(!node.allocated);
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +05301390 }
Daniel Vetter935aaa62012-03-25 19:47:35 +02001391
1392 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1393 if (ret)
1394 goto out_unpin;
1395
Chris Wilsonfe115622016-10-28 13:58:40 +01001396 mutex_unlock(&i915->drm.struct_mutex);
1397
Chris Wilsonb19482d2016-08-18 17:16:43 +01001398 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
Paulo Zanoni063e4e62015-02-13 17:23:45 -02001399
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +05301400 user_data = u64_to_user_ptr(args->data_ptr);
1401 offset = args->offset;
1402 remain = args->size;
1403 while (remain) {
Eric Anholt673a3942008-07-30 12:06:12 -07001404 /* Operation in this page
1405 *
Keith Packard0839ccb2008-10-30 19:38:48 -07001406 * page_base = page offset within aperture
1407 * page_offset = offset within page
1408 * page_length = bytes to copy for this page
Eric Anholt673a3942008-07-30 12:06:12 -07001409 */
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +05301410 u32 page_base = node.start;
Chris Wilsonbb6dc8d2016-10-28 13:58:39 +01001411 unsigned int page_offset = offset_in_page(offset);
1412 unsigned int page_length = PAGE_SIZE - page_offset;
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +05301413 page_length = remain < page_length ? remain : page_length;
1414 if (node.allocated) {
1415 wmb(); /* flush the write before we modify the GGTT */
1416 ggtt->base.insert_page(&ggtt->base,
1417 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
1418 node.start, I915_CACHE_NONE, 0);
1419 wmb(); /* flush modifications to the GGTT (insert_page) */
1420 } else {
1421 page_base += offset & PAGE_MASK;
1422 }
Keith Packard0839ccb2008-10-30 19:38:48 -07001423 /* If we get a fault while copying data, then (presumably) our
Eric Anholt3de09aa2009-03-09 09:42:23 -07001424 * source page isn't available. Return the error and we'll
1425 * retry in the slow path.
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301426 * If the object is non-shmem backed, we retry again with the
1427 * path that handles page fault.
Keith Packard0839ccb2008-10-30 19:38:48 -07001428 */
Matthew Auld73ebd502017-12-11 15:18:20 +00001429 if (ggtt_write(&ggtt->iomap, page_base, page_offset,
Chris Wilsonfe115622016-10-28 13:58:40 +01001430 user_data, page_length)) {
1431 ret = -EFAULT;
1432 break;
Daniel Vetter935aaa62012-03-25 19:47:35 +02001433 }
Eric Anholt673a3942008-07-30 12:06:12 -07001434
Keith Packard0839ccb2008-10-30 19:38:48 -07001435 remain -= page_length;
1436 user_data += page_length;
1437 offset += page_length;
Eric Anholt673a3942008-07-30 12:06:12 -07001438 }
Chris Wilsond59b21e2017-02-22 11:40:49 +00001439 intel_fb_obj_flush(obj, ORIGIN_CPU);
Chris Wilsonfe115622016-10-28 13:58:40 +01001440
1441 mutex_lock(&i915->drm.struct_mutex);
Daniel Vetter935aaa62012-03-25 19:47:35 +02001442out_unpin:
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +05301443 if (node.allocated) {
1444 wmb();
1445 ggtt->base.clear_range(&ggtt->base,
Michał Winiarski4fb84d92016-10-13 14:02:40 +02001446 node.start, node.size);
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +05301447 remove_mappable_node(&node);
1448 } else {
Chris Wilson058d88c2016-08-15 10:49:06 +01001449 i915_vma_unpin(vma);
Ankitprasad Sharma4f1959e2016-06-10 14:23:01 +05301450 }
Chris Wilson8bd81812017-10-19 07:37:33 +01001451out_rpm:
Chris Wilson9c870d02016-10-24 13:42:15 +01001452 intel_runtime_pm_put(i915);
Chris Wilson8bd81812017-10-19 07:37:33 +01001453out_unlock:
Chris Wilsonfe115622016-10-28 13:58:40 +01001454 mutex_unlock(&i915->drm.struct_mutex);
Eric Anholt3de09aa2009-03-09 09:42:23 -07001455 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07001456}
1457
Eric Anholt673a3942008-07-30 12:06:12 -07001458static int
Chris Wilsonfe115622016-10-28 13:58:40 +01001459shmem_pwrite_slow(struct page *page, int offset, int length,
Daniel Vetterd174bd62012-03-25 19:47:40 +02001460 char __user *user_data,
1461 bool page_do_bit17_swizzling,
1462 bool needs_clflush_before,
1463 bool needs_clflush_after)
Eric Anholt673a3942008-07-30 12:06:12 -07001464{
Daniel Vetterd174bd62012-03-25 19:47:40 +02001465 char *vaddr;
1466 int ret;
Eric Anholt40123c12009-03-09 13:42:30 -07001467
Daniel Vetterd174bd62012-03-25 19:47:40 +02001468 vaddr = kmap(page);
Daniel Vettere7e58eb2012-03-25 19:47:43 +02001469 if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
Chris Wilsonfe115622016-10-28 13:58:40 +01001470 shmem_clflush_swizzled_range(vaddr + offset, length,
Daniel Vetter23c18c72012-03-25 19:47:42 +02001471 page_do_bit17_swizzling);
Daniel Vetterd174bd62012-03-25 19:47:40 +02001472 if (page_do_bit17_swizzling)
Chris Wilsonfe115622016-10-28 13:58:40 +01001473 ret = __copy_from_user_swizzled(vaddr, offset, user_data,
1474 length);
Daniel Vetterd174bd62012-03-25 19:47:40 +02001475 else
Chris Wilsonfe115622016-10-28 13:58:40 +01001476 ret = __copy_from_user(vaddr + offset, user_data, length);
Daniel Vetterd174bd62012-03-25 19:47:40 +02001477 if (needs_clflush_after)
Chris Wilsonfe115622016-10-28 13:58:40 +01001478 shmem_clflush_swizzled_range(vaddr + offset, length,
Daniel Vetter23c18c72012-03-25 19:47:42 +02001479 page_do_bit17_swizzling);
Daniel Vetterd174bd62012-03-25 19:47:40 +02001480 kunmap(page);
Chris Wilsone5281cc2010-10-28 13:45:36 +01001481
Chris Wilson755d2212012-09-04 21:02:55 +01001482 return ret ? -EFAULT : 0;
Eric Anholt40123c12009-03-09 13:42:30 -07001483}
1484
Chris Wilsonfe115622016-10-28 13:58:40 +01001485/* Per-page copy function for the shmem pwrite fastpath.
1486 * Flushes invalid cachelines before writing to the target if
1487 * needs_clflush_before is set and flushes out any written cachelines after
1488 * writing if needs_clflush is set.
1489 */
Eric Anholt40123c12009-03-09 13:42:30 -07001490static int
Chris Wilsonfe115622016-10-28 13:58:40 +01001491shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
1492 bool page_do_bit17_swizzling,
1493 bool needs_clflush_before,
1494 bool needs_clflush_after)
Eric Anholt40123c12009-03-09 13:42:30 -07001495{
Chris Wilsonfe115622016-10-28 13:58:40 +01001496 int ret;
Eric Anholt40123c12009-03-09 13:42:30 -07001497
Chris Wilsonfe115622016-10-28 13:58:40 +01001498 ret = -ENODEV;
1499 if (!page_do_bit17_swizzling) {
1500 char *vaddr = kmap_atomic(page);
1501
1502 if (needs_clflush_before)
1503 drm_clflush_virt_range(vaddr + offset, len);
1504 ret = __copy_from_user_inatomic(vaddr + offset, user_data, len);
1505 if (needs_clflush_after)
1506 drm_clflush_virt_range(vaddr + offset, len);
1507
1508 kunmap_atomic(vaddr);
1509 }
1510 if (ret == 0)
1511 return ret;
1512
1513 return shmem_pwrite_slow(page, offset, len, user_data,
1514 page_do_bit17_swizzling,
1515 needs_clflush_before,
1516 needs_clflush_after);
1517}
1518
1519static int
1520i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
1521 const struct drm_i915_gem_pwrite *args)
1522{
1523 struct drm_i915_private *i915 = to_i915(obj->base.dev);
1524 void __user *user_data;
1525 u64 remain;
1526 unsigned int obj_do_bit17_swizzling;
1527 unsigned int partial_cacheline_write;
1528 unsigned int needs_clflush;
1529 unsigned int offset, idx;
1530 int ret;
1531
1532 ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
Chris Wilson43394c72016-08-18 17:16:47 +01001533 if (ret)
1534 return ret;
Eric Anholt40123c12009-03-09 13:42:30 -07001535
Chris Wilsonfe115622016-10-28 13:58:40 +01001536 ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
1537 mutex_unlock(&i915->drm.struct_mutex);
1538 if (ret)
1539 return ret;
1540
1541 obj_do_bit17_swizzling = 0;
1542 if (i915_gem_object_needs_bit17_swizzle(obj))
1543 obj_do_bit17_swizzling = BIT(17);
1544
1545 /* If we don't overwrite a cacheline completely we need to be
1546 * careful to have up-to-date data by first clflushing. Don't
1547 * overcomplicate things and flush the entire patch.
1548 */
1549 partial_cacheline_write = 0;
1550 if (needs_clflush & CLFLUSH_BEFORE)
1551 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
1552
Chris Wilson43394c72016-08-18 17:16:47 +01001553 user_data = u64_to_user_ptr(args->data_ptr);
Chris Wilson43394c72016-08-18 17:16:47 +01001554 remain = args->size;
Chris Wilsonfe115622016-10-28 13:58:40 +01001555 offset = offset_in_page(args->offset);
1556 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
1557 struct page *page = i915_gem_object_get_page(obj, idx);
1558 int length;
Eric Anholt40123c12009-03-09 13:42:30 -07001559
Chris Wilsonfe115622016-10-28 13:58:40 +01001560 length = remain;
1561 if (offset + length > PAGE_SIZE)
1562 length = PAGE_SIZE - offset;
Chris Wilsone5281cc2010-10-28 13:45:36 +01001563
Chris Wilsonfe115622016-10-28 13:58:40 +01001564 ret = shmem_pwrite(page, offset, length, user_data,
1565 page_to_phys(page) & obj_do_bit17_swizzling,
1566 (offset | length) & partial_cacheline_write,
1567 needs_clflush & CLFLUSH_AFTER);
1568 if (ret)
Chris Wilson9da3da62012-06-01 15:20:22 +01001569 break;
1570
Chris Wilsonfe115622016-10-28 13:58:40 +01001571 remain -= length;
1572 user_data += length;
1573 offset = 0;
Eric Anholt40123c12009-03-09 13:42:30 -07001574 }
1575
Chris Wilsond59b21e2017-02-22 11:40:49 +00001576 intel_fb_obj_flush(obj, ORIGIN_CPU);
Chris Wilsonfe115622016-10-28 13:58:40 +01001577 i915_gem_obj_finish_shmem_access(obj);
Eric Anholt40123c12009-03-09 13:42:30 -07001578 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07001579}
1580
1581/**
1582 * Writes data to the object referenced by handle.
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +01001583 * @dev: drm device
1584 * @data: ioctl data blob
1585 * @file: drm file
Eric Anholt673a3942008-07-30 12:06:12 -07001586 *
1587 * On error, the contents of the buffer that were to be modified are undefined.
1588 */
1589int
1590i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
Chris Wilsonfbd5a262010-10-14 15:03:58 +01001591 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001592{
1593 struct drm_i915_gem_pwrite *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00001594 struct drm_i915_gem_object *obj;
Chris Wilson51311d02010-11-17 09:10:42 +00001595 int ret;
1596
1597 if (args->size == 0)
1598 return 0;
1599
1600 if (!access_ok(VERIFY_READ,
Gustavo Padovan3ed605b2016-04-26 12:32:27 -03001601 u64_to_user_ptr(args->data_ptr),
Chris Wilson51311d02010-11-17 09:10:42 +00001602 args->size))
1603 return -EFAULT;
1604
Chris Wilson03ac0642016-07-20 13:31:51 +01001605 obj = i915_gem_object_lookup(file, args->handle);
Chris Wilson258a5ed2016-08-05 10:14:16 +01001606 if (!obj)
1607 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07001608
Chris Wilson7dcd2492010-09-26 20:21:44 +01001609 /* Bounds check destination. */
Matthew Auld966d5bf2016-12-13 20:32:22 +00001610 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
Chris Wilsonce9d4192010-09-26 20:50:05 +01001611 ret = -EINVAL;
Chris Wilson258a5ed2016-08-05 10:14:16 +01001612 goto err;
Chris Wilsonce9d4192010-09-26 20:50:05 +01001613 }
1614
Chris Wilsondb53a302011-02-03 11:57:46 +00001615 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1616
Chris Wilson7c55e2c2017-03-07 12:03:38 +00001617 ret = -ENODEV;
1618 if (obj->ops->pwrite)
1619 ret = obj->ops->pwrite(obj, args);
1620 if (ret != -ENODEV)
1621 goto err;
1622
Chris Wilsone95433c2016-10-28 13:58:27 +01001623 ret = i915_gem_object_wait(obj,
1624 I915_WAIT_INTERRUPTIBLE |
1625 I915_WAIT_ALL,
1626 MAX_SCHEDULE_TIMEOUT,
1627 to_rps_client(file));
Chris Wilson258a5ed2016-08-05 10:14:16 +01001628 if (ret)
1629 goto err;
1630
Chris Wilsonfe115622016-10-28 13:58:40 +01001631 ret = i915_gem_object_pin_pages(obj);
Chris Wilson258a5ed2016-08-05 10:14:16 +01001632 if (ret)
Chris Wilsonfe115622016-10-28 13:58:40 +01001633 goto err;
Chris Wilson258a5ed2016-08-05 10:14:16 +01001634
Daniel Vetter935aaa62012-03-25 19:47:35 +02001635 ret = -EFAULT;
Eric Anholt673a3942008-07-30 12:06:12 -07001636 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1637 * it would end up going through the fenced access, and we'll get
1638 * different detiling behavior between reading and writing.
1639 * pread/pwrite currently are reading and writing from the CPU
1640 * perspective, requiring manual detiling by the client.
1641 */
Chris Wilson6eae0052016-06-20 15:05:52 +01001642 if (!i915_gem_object_has_struct_page(obj) ||
Chris Wilson9c870d02016-10-24 13:42:15 +01001643 cpu_write_needs_clflush(obj))
Daniel Vetter935aaa62012-03-25 19:47:35 +02001644 /* Note that the gtt paths might fail with non-page-backed user
1645 * pointers (e.g. gtt mappings when moving data between
Chris Wilson9c870d02016-10-24 13:42:15 +01001646 * textures). Fallback to the shmem path in that case.
1647 */
Chris Wilsonfe115622016-10-28 13:58:40 +01001648 ret = i915_gem_gtt_pwrite_fast(obj, args);
Eric Anholt673a3942008-07-30 12:06:12 -07001649
Chris Wilsond1054ee2016-07-16 18:42:36 +01001650 if (ret == -EFAULT || ret == -ENOSPC) {
Chris Wilson6a2c4232014-11-04 04:51:40 -08001651 if (obj->phys_handle)
1652 ret = i915_gem_phys_pwrite(obj, args, file);
Ankitprasad Sharmab50a5372016-06-10 14:23:03 +05301653 else
Chris Wilsonfe115622016-10-28 13:58:40 +01001654 ret = i915_gem_shmem_pwrite(obj, args);
Chris Wilson6a2c4232014-11-04 04:51:40 -08001655 }
Daniel Vetter5c0480f2011-12-14 13:57:30 +01001656
Chris Wilsonfe115622016-10-28 13:58:40 +01001657 i915_gem_object_unpin_pages(obj);
Chris Wilson258a5ed2016-08-05 10:14:16 +01001658err:
Chris Wilsonf0cd5182016-10-28 13:58:43 +01001659 i915_gem_object_put(obj);
Chris Wilson258a5ed2016-08-05 10:14:16 +01001660 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07001661}
1662
Chris Wilson40e62d52016-10-28 13:58:41 +01001663static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
1664{
1665 struct drm_i915_private *i915;
1666 struct list_head *list;
1667 struct i915_vma *vma;
1668
Chris Wilsonf2123812017-10-16 12:40:37 +01001669 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
1670
Chris Wilsone2189dd2017-12-07 21:14:07 +00001671 for_each_ggtt_vma(vma, obj) {
Chris Wilson40e62d52016-10-28 13:58:41 +01001672 if (i915_vma_is_active(vma))
1673 continue;
1674
1675 if (!drm_mm_node_allocated(&vma->node))
1676 continue;
1677
1678 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
1679 }
1680
1681 i915 = to_i915(obj->base.dev);
Chris Wilsonf2123812017-10-16 12:40:37 +01001682 spin_lock(&i915->mm.obj_lock);
Chris Wilson40e62d52016-10-28 13:58:41 +01001683 list = obj->bind_count ? &i915->mm.bound_list : &i915->mm.unbound_list;
Chris Wilsonf2123812017-10-16 12:40:37 +01001684 list_move_tail(&obj->mm.link, list);
1685 spin_unlock(&i915->mm.obj_lock);
Chris Wilson40e62d52016-10-28 13:58:41 +01001686}
1687
Eric Anholt673a3942008-07-30 12:06:12 -07001688/**
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001689 * Called when user space prepares to use an object with the CPU, either
1690 * through the mmap ioctl's mapping or a GTT mapping.
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +01001691 * @dev: drm device
1692 * @data: ioctl data blob
1693 * @file: drm file
Eric Anholt673a3942008-07-30 12:06:12 -07001694 */
1695int
1696i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001697 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001698{
1699 struct drm_i915_gem_set_domain *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00001700 struct drm_i915_gem_object *obj;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001701 uint32_t read_domains = args->read_domains;
1702 uint32_t write_domain = args->write_domain;
Chris Wilson40e62d52016-10-28 13:58:41 +01001703 int err;
Eric Anholt673a3942008-07-30 12:06:12 -07001704
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001705 /* Only handle setting domains to types used by the CPU. */
Chris Wilsonb8f90962016-08-05 10:14:07 +01001706 if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001707 return -EINVAL;
1708
1709 /* Having something in the write domain implies it's in the read
1710 * domain, and only that read domain. Enforce that in the request.
1711 */
1712 if (write_domain != 0 && read_domains != write_domain)
1713 return -EINVAL;
1714
Chris Wilson03ac0642016-07-20 13:31:51 +01001715 obj = i915_gem_object_lookup(file, args->handle);
Chris Wilsonb8f90962016-08-05 10:14:07 +01001716 if (!obj)
1717 return -ENOENT;
Jesse Barnes652c3932009-08-17 13:31:43 -07001718
Chris Wilson3236f572012-08-24 09:35:09 +01001719 /* Try to flush the object off the GPU without holding the lock.
1720 * We will repeat the flush holding the lock in the normal manner
1721 * to catch cases where we are gazumped.
1722 */
Chris Wilson40e62d52016-10-28 13:58:41 +01001723 err = i915_gem_object_wait(obj,
Chris Wilsone95433c2016-10-28 13:58:27 +01001724 I915_WAIT_INTERRUPTIBLE |
1725 (write_domain ? I915_WAIT_ALL : 0),
1726 MAX_SCHEDULE_TIMEOUT,
1727 to_rps_client(file));
Chris Wilson40e62d52016-10-28 13:58:41 +01001728 if (err)
Chris Wilsonf0cd5182016-10-28 13:58:43 +01001729 goto out;
Chris Wilsonb8f90962016-08-05 10:14:07 +01001730
Tina Zhanga03f3952017-11-14 10:25:13 +00001731 /*
1732 * Proxy objects do not control access to the backing storage, ergo
1733 * they cannot be used as a means to manipulate the cache domain
1734 * tracking for that backing storage. The proxy object is always
1735 * considered to be outside of any cache domain.
1736 */
1737 if (i915_gem_object_is_proxy(obj)) {
1738 err = -ENXIO;
1739 goto out;
1740 }
1741
1742 /*
1743 * Flush and acquire obj->pages so that we are coherent through
Chris Wilson40e62d52016-10-28 13:58:41 +01001744 * direct access in memory with previous cached writes through
1745 * shmemfs and that our cache domain tracking remains valid.
1746 * For example, if the obj->filp was moved to swap without us
1747 * being notified and releasing the pages, we would mistakenly
1748 * continue to assume that the obj remained out of the CPU cached
1749 * domain.
1750 */
1751 err = i915_gem_object_pin_pages(obj);
1752 if (err)
Chris Wilsonf0cd5182016-10-28 13:58:43 +01001753 goto out;
Chris Wilson40e62d52016-10-28 13:58:41 +01001754
1755 err = i915_mutex_lock_interruptible(dev);
1756 if (err)
Chris Wilsonf0cd5182016-10-28 13:58:43 +01001757 goto out_unpin;
Chris Wilson3236f572012-08-24 09:35:09 +01001758
Chris Wilsone22d8e32017-04-12 12:01:11 +01001759 if (read_domains & I915_GEM_DOMAIN_WC)
1760 err = i915_gem_object_set_to_wc_domain(obj, write_domain);
1761 else if (read_domains & I915_GEM_DOMAIN_GTT)
1762 err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
Chris Wilson43566de2015-01-02 16:29:29 +05301763 else
Chris Wilsone22d8e32017-04-12 12:01:11 +01001764 err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
Chris Wilson40e62d52016-10-28 13:58:41 +01001765
1766 /* And bump the LRU for this access */
1767 i915_gem_object_bump_inactive_ggtt(obj);
1768
1769 mutex_unlock(&dev->struct_mutex);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08001770
Daniel Vetter031b6982015-06-26 19:35:16 +02001771 if (write_domain != 0)
Chris Wilsonef749212017-04-12 12:01:10 +01001772 intel_fb_obj_invalidate(obj,
1773 fb_write_origin(obj, write_domain));
Daniel Vetter031b6982015-06-26 19:35:16 +02001774
Chris Wilsonf0cd5182016-10-28 13:58:43 +01001775out_unpin:
Chris Wilson40e62d52016-10-28 13:58:41 +01001776 i915_gem_object_unpin_pages(obj);
Chris Wilsonf0cd5182016-10-28 13:58:43 +01001777out:
1778 i915_gem_object_put(obj);
Chris Wilson40e62d52016-10-28 13:58:41 +01001779 return err;
Eric Anholt673a3942008-07-30 12:06:12 -07001780}
1781
1782/**
1783 * Called when user space has done writes to this buffer
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +01001784 * @dev: drm device
1785 * @data: ioctl data blob
1786 * @file: drm file
Eric Anholt673a3942008-07-30 12:06:12 -07001787 */
1788int
1789i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001790 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001791{
1792 struct drm_i915_gem_sw_finish *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00001793 struct drm_i915_gem_object *obj;
Chris Wilson1d7cfea2010-10-17 09:45:41 +01001794
Chris Wilson03ac0642016-07-20 13:31:51 +01001795 obj = i915_gem_object_lookup(file, args->handle);
Chris Wilsonc21724c2016-08-05 10:14:19 +01001796 if (!obj)
1797 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07001798
Tina Zhanga03f3952017-11-14 10:25:13 +00001799 /*
1800 * Proxy objects are barred from CPU access, so there is no
1801 * need to ban sw_finish as it is a nop.
1802 */
1803
Eric Anholt673a3942008-07-30 12:06:12 -07001804 /* Pinned buffers may be scanout, so flush the cache */
Chris Wilson5a97bcc2017-02-22 11:40:46 +00001805 i915_gem_object_flush_if_display(obj);
Chris Wilsonf0cd5182016-10-28 13:58:43 +01001806 i915_gem_object_put(obj);
Chris Wilson5a97bcc2017-02-22 11:40:46 +00001807
1808 return 0;
Eric Anholt673a3942008-07-30 12:06:12 -07001809}
1810
1811/**
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +01001812 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
1813 * it is mapped to.
1814 * @dev: drm device
1815 * @data: ioctl data blob
1816 * @file: drm file
Eric Anholt673a3942008-07-30 12:06:12 -07001817 *
1818 * While the mapping holds a reference on the contents of the object, it doesn't
1819 * imply a ref on the object itself.
Daniel Vetter34367382014-10-16 12:28:18 +02001820 *
1821 * IMPORTANT:
1822 *
1823 * DRM driver writers who look a this function as an example for how to do GEM
1824 * mmap support, please don't implement mmap support like here. The modern way
1825 * to implement DRM mmap support is with an mmap offset ioctl (like
1826 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1827 * That way debug tooling like valgrind will understand what's going on, hiding
1828 * the mmap call in a driver private ioctl will break that. The i915 driver only
1829 * does cpu mmaps this way because we didn't know better.
Eric Anholt673a3942008-07-30 12:06:12 -07001830 */
1831int
1832i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00001833 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07001834{
1835 struct drm_i915_gem_mmap *args = data;
Chris Wilson03ac0642016-07-20 13:31:51 +01001836 struct drm_i915_gem_object *obj;
Eric Anholt673a3942008-07-30 12:06:12 -07001837 unsigned long addr;
1838
Akash Goel1816f922015-01-02 16:29:30 +05301839 if (args->flags & ~(I915_MMAP_WC))
1840 return -EINVAL;
1841
Borislav Petkov568a58e2016-03-29 17:42:01 +02001842 if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
Akash Goel1816f922015-01-02 16:29:30 +05301843 return -ENODEV;
1844
Chris Wilson03ac0642016-07-20 13:31:51 +01001845 obj = i915_gem_object_lookup(file, args->handle);
1846 if (!obj)
Chris Wilsonbf79cb92010-08-04 14:19:46 +01001847 return -ENOENT;
Eric Anholt673a3942008-07-30 12:06:12 -07001848
Daniel Vetter1286ff72012-05-10 15:25:09 +02001849 /* prime objects have no backing filp to GEM mmap
1850 * pages from.
1851 */
Chris Wilson03ac0642016-07-20 13:31:51 +01001852 if (!obj->base.filp) {
Chris Wilsonf0cd5182016-10-28 13:58:43 +01001853 i915_gem_object_put(obj);
Tina Zhang274b2462017-11-14 10:25:12 +00001854 return -ENXIO;
Daniel Vetter1286ff72012-05-10 15:25:09 +02001855 }
1856
Chris Wilson03ac0642016-07-20 13:31:51 +01001857 addr = vm_mmap(obj->base.filp, 0, args->size,
Eric Anholt673a3942008-07-30 12:06:12 -07001858 PROT_READ | PROT_WRITE, MAP_SHARED,
1859 args->offset);
Akash Goel1816f922015-01-02 16:29:30 +05301860 if (args->flags & I915_MMAP_WC) {
1861 struct mm_struct *mm = current->mm;
1862 struct vm_area_struct *vma;
1863
Michal Hocko80a89a52016-05-23 16:26:11 -07001864 if (down_write_killable(&mm->mmap_sem)) {
Chris Wilsonf0cd5182016-10-28 13:58:43 +01001865 i915_gem_object_put(obj);
Michal Hocko80a89a52016-05-23 16:26:11 -07001866 return -EINTR;
1867 }
Akash Goel1816f922015-01-02 16:29:30 +05301868 vma = find_vma(mm, addr);
1869 if (vma)
1870 vma->vm_page_prot =
1871 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1872 else
1873 addr = -ENOMEM;
1874 up_write(&mm->mmap_sem);
Chris Wilsonaeecc962016-06-17 14:46:39 -03001875
1876 /* This may race, but that's ok, it only gets set */
Chris Wilson50349242016-08-18 17:17:04 +01001877 WRITE_ONCE(obj->frontbuffer_ggtt_origin, ORIGIN_CPU);
Akash Goel1816f922015-01-02 16:29:30 +05301878 }
Chris Wilsonf0cd5182016-10-28 13:58:43 +01001879 i915_gem_object_put(obj);
Eric Anholt673a3942008-07-30 12:06:12 -07001880 if (IS_ERR((void *)addr))
1881 return addr;
1882
1883 args->addr_ptr = (uint64_t) addr;
1884
1885 return 0;
1886}
1887
Chris Wilson03af84f2016-08-18 17:17:01 +01001888static unsigned int tile_row_pages(struct drm_i915_gem_object *obj)
1889{
Chris Wilson6649a0b2017-01-09 16:16:08 +00001890 return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;
Chris Wilson03af84f2016-08-18 17:17:01 +01001891}
1892
Jesse Barnesde151cf2008-11-12 10:03:55 -08001893/**
Chris Wilson4cc69072016-08-25 19:05:19 +01001894 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
1895 *
1896 * A history of the GTT mmap interface:
1897 *
1898 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
1899 * aligned and suitable for fencing, and still fit into the available
1900 * mappable space left by the pinned display objects. A classic problem
1901 * we called the page-fault-of-doom where we would ping-pong between
1902 * two objects that could not fit inside the GTT and so the memcpy
1903 * would page one object in at the expense of the other between every
1904 * single byte.
1905 *
1906 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
1907 * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
1908 * object is too large for the available space (or simply too large
1909 * for the mappable aperture!), a view is created instead and faulted
1910 * into userspace. (This view is aligned and sized appropriately for
1911 * fenced access.)
1912 *
Chris Wilsone22d8e32017-04-12 12:01:11 +01001913 * 2 - Recognise WC as a separate cache domain so that we can flush the
1914 * delayed writes via GTT before performing direct access via WC.
1915 *
Chris Wilson4cc69072016-08-25 19:05:19 +01001916 * Restrictions:
1917 *
1918 * * snoopable objects cannot be accessed via the GTT. It can cause machine
1919 * hangs on some architectures, corruption on others. An attempt to service
1920 * a GTT page fault from a snoopable object will generate a SIGBUS.
1921 *
1922 * * the object must be able to fit into RAM (physical memory, though no
1923 * limited to the mappable aperture).
1924 *
1925 *
1926 * Caveats:
1927 *
1928 * * a new GTT page fault will synchronize rendering from the GPU and flush
1929 * all data to system memory. Subsequent access will not be synchronized.
1930 *
1931 * * all mappings are revoked on runtime device suspend.
1932 *
1933 * * there are only 8, 16 or 32 fence registers to share between all users
1934 * (older machines require fence register for display and blitter access
1935 * as well). Contention of the fence registers will cause the previous users
1936 * to be unmapped and any new access will generate new page faults.
1937 *
1938 * * running out of memory while servicing a fault may generate a SIGBUS,
1939 * rather than the expected SIGSEGV.
1940 */
1941int i915_gem_mmap_gtt_version(void)
1942{
Chris Wilsone22d8e32017-04-12 12:01:11 +01001943 return 2;
Chris Wilson4cc69072016-08-25 19:05:19 +01001944}
1945
Chris Wilson2d4281b2017-01-10 09:56:32 +00001946static inline struct i915_ggtt_view
1947compute_partial_view(struct drm_i915_gem_object *obj,
Chris Wilson2d4281b2017-01-10 09:56:32 +00001948 pgoff_t page_offset,
1949 unsigned int chunk)
1950{
1951 struct i915_ggtt_view view;
1952
1953 if (i915_gem_object_is_tiled(obj))
1954 chunk = roundup(chunk, tile_row_pages(obj));
1955
Chris Wilson2d4281b2017-01-10 09:56:32 +00001956 view.type = I915_GGTT_VIEW_PARTIAL;
Chris Wilson8bab11932017-01-14 00:28:25 +00001957 view.partial.offset = rounddown(page_offset, chunk);
1958 view.partial.size =
Chris Wilson2d4281b2017-01-10 09:56:32 +00001959 min_t(unsigned int, chunk,
Chris Wilson8bab11932017-01-14 00:28:25 +00001960 (obj->base.size >> PAGE_SHIFT) - view.partial.offset);
Chris Wilson2d4281b2017-01-10 09:56:32 +00001961
1962 /* If the partial covers the entire object, just create a normal VMA. */
1963 if (chunk >= obj->base.size >> PAGE_SHIFT)
1964 view.type = I915_GGTT_VIEW_NORMAL;
1965
1966 return view;
1967}
1968
Chris Wilson4cc69072016-08-25 19:05:19 +01001969/**
Jesse Barnesde151cf2008-11-12 10:03:55 -08001970 * i915_gem_fault - fault a page into the GTT
Geliang Tangd9072a32015-09-15 05:58:44 -07001971 * @vmf: fault info
Jesse Barnesde151cf2008-11-12 10:03:55 -08001972 *
1973 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1974 * from userspace. The fault handler takes care of binding the object to
1975 * the GTT (if needed), allocating and programming a fence register (again,
1976 * only if needed based on whether the old reg is still valid or the object
1977 * is tiled) and inserting a new PTE into the faulting process.
1978 *
1979 * Note that the faulting process may involve evicting existing objects
1980 * from the GTT and/or fence registers to make room. So performance may
1981 * suffer if the GTT working set is large or there are few fence registers
1982 * left.
Chris Wilson4cc69072016-08-25 19:05:19 +01001983 *
1984 * The current feature set supported by i915_gem_fault() and thus GTT mmaps
1985 * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version).
Jesse Barnesde151cf2008-11-12 10:03:55 -08001986 */
Dave Jiang11bac802017-02-24 14:56:41 -08001987int i915_gem_fault(struct vm_fault *vmf)
Jesse Barnesde151cf2008-11-12 10:03:55 -08001988{
Chris Wilson03af84f2016-08-18 17:17:01 +01001989#define MIN_CHUNK_PAGES ((1 << 20) >> PAGE_SHIFT) /* 1 MiB */
Dave Jiang11bac802017-02-24 14:56:41 -08001990 struct vm_area_struct *area = vmf->vma;
Chris Wilson058d88c2016-08-15 10:49:06 +01001991 struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
Chris Wilson05394f32010-11-08 19:18:58 +00001992 struct drm_device *dev = obj->base.dev;
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03001993 struct drm_i915_private *dev_priv = to_i915(dev);
1994 struct i915_ggtt *ggtt = &dev_priv->ggtt;
Chris Wilsonb8f90962016-08-05 10:14:07 +01001995 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
Chris Wilson058d88c2016-08-15 10:49:06 +01001996 struct i915_vma *vma;
Jesse Barnesde151cf2008-11-12 10:03:55 -08001997 pgoff_t page_offset;
Chris Wilson82118872016-08-18 17:17:05 +01001998 unsigned int flags;
Chris Wilsonb8f90962016-08-05 10:14:07 +01001999 int ret;
Paulo Zanonif65c9162013-11-27 18:20:34 -02002000
Jesse Barnesde151cf2008-11-12 10:03:55 -08002001 /* We don't use vmf->pgoff since that has the fake offset */
Jan Kara1a29d852016-12-14 15:07:01 -08002002 page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002003
Chris Wilsondb53a302011-02-03 11:57:46 +00002004 trace_i915_gem_object_fault(obj, page_offset, true, write);
2005
Chris Wilson6e4930f2014-02-07 18:37:06 -02002006 /* Try to flush the object off the GPU first without holding the lock.
Chris Wilsonb8f90962016-08-05 10:14:07 +01002007 * Upon acquiring the lock, we will perform our sanity checks and then
Chris Wilson6e4930f2014-02-07 18:37:06 -02002008 * repeat the flush holding the lock in the normal manner to catch cases
2009 * where we are gazumped.
2010 */
Chris Wilsone95433c2016-10-28 13:58:27 +01002011 ret = i915_gem_object_wait(obj,
2012 I915_WAIT_INTERRUPTIBLE,
2013 MAX_SCHEDULE_TIMEOUT,
2014 NULL);
Chris Wilson6e4930f2014-02-07 18:37:06 -02002015 if (ret)
Chris Wilsonb8f90962016-08-05 10:14:07 +01002016 goto err;
2017
Chris Wilson40e62d52016-10-28 13:58:41 +01002018 ret = i915_gem_object_pin_pages(obj);
2019 if (ret)
2020 goto err;
2021
Chris Wilsonb8f90962016-08-05 10:14:07 +01002022 intel_runtime_pm_get(dev_priv);
2023
2024 ret = i915_mutex_lock_interruptible(dev);
2025 if (ret)
2026 goto err_rpm;
Chris Wilson6e4930f2014-02-07 18:37:06 -02002027
Chris Wilsoneb119bd2012-12-16 12:43:36 +00002028 /* Access to snoopable pages through the GTT is incoherent. */
Tvrtko Ursulin0031fb92016-11-04 14:42:44 +00002029 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv)) {
Chris Wilsonddeff6e2014-05-28 16:16:41 +01002030 ret = -EFAULT;
Chris Wilsonb8f90962016-08-05 10:14:07 +01002031 goto err_unlock;
Chris Wilsoneb119bd2012-12-16 12:43:36 +00002032 }
2033
Chris Wilson82118872016-08-18 17:17:05 +01002034 /* If the object is smaller than a couple of partial vma, it is
2035 * not worth only creating a single partial vma - we may as well
2036 * clear enough space for the full object.
2037 */
2038 flags = PIN_MAPPABLE;
2039 if (obj->base.size > 2 * MIN_CHUNK_PAGES << PAGE_SHIFT)
2040 flags |= PIN_NONBLOCK | PIN_NONFAULT;
2041
Chris Wilsona61007a2016-08-18 17:17:02 +01002042 /* Now pin it into the GTT as needed */
Chris Wilson82118872016-08-18 17:17:05 +01002043 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, flags);
Chris Wilsona61007a2016-08-18 17:17:02 +01002044 if (IS_ERR(vma)) {
Chris Wilsona61007a2016-08-18 17:17:02 +01002045 /* Use a partial view if it is bigger than available space */
Chris Wilson2d4281b2017-01-10 09:56:32 +00002046 struct i915_ggtt_view view =
Chris Wilson8201c1f2017-01-10 09:56:33 +00002047 compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
Chris Wilsonaa136d92016-08-18 17:17:03 +01002048
Chris Wilson50349242016-08-18 17:17:04 +01002049 /* Userspace is now writing through an untracked VMA, abandon
2050 * all hope that the hardware is able to track future writes.
2051 */
2052 obj->frontbuffer_ggtt_origin = ORIGIN_CPU;
2053
Chris Wilsona61007a2016-08-18 17:17:02 +01002054 vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
2055 }
Chris Wilson058d88c2016-08-15 10:49:06 +01002056 if (IS_ERR(vma)) {
2057 ret = PTR_ERR(vma);
Chris Wilsonb8f90962016-08-05 10:14:07 +01002058 goto err_unlock;
Chris Wilson058d88c2016-08-15 10:49:06 +01002059 }
Jesse Barnesde151cf2008-11-12 10:03:55 -08002060
Chris Wilsonc9839302012-11-20 10:45:17 +00002061 ret = i915_gem_object_set_to_gtt_domain(obj, write);
2062 if (ret)
Chris Wilsonb8f90962016-08-05 10:14:07 +01002063 goto err_unpin;
Chris Wilsonc9839302012-11-20 10:45:17 +00002064
Chris Wilson3bd40732017-10-09 09:43:56 +01002065 ret = i915_vma_pin_fence(vma);
Chris Wilsonc9839302012-11-20 10:45:17 +00002066 if (ret)
Chris Wilsonb8f90962016-08-05 10:14:07 +01002067 goto err_unpin;
Chris Wilson7d1c4802010-08-07 21:45:03 +01002068
Chris Wilsonb90b91d2014-06-10 12:14:40 +01002069 /* Finally, remap it using the new GTT offset */
Chris Wilsonc58305a2016-08-19 16:54:28 +01002070 ret = remap_io_mapping(area,
Chris Wilson8bab11932017-01-14 00:28:25 +00002071 area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT),
Matthew Auld73ebd502017-12-11 15:18:20 +00002072 (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT,
Chris Wilsonc58305a2016-08-19 16:54:28 +01002073 min_t(u64, vma->size, area->vm_end - area->vm_start),
Matthew Auld73ebd502017-12-11 15:18:20 +00002074 &ggtt->iomap);
Chris Wilsona65adaf2017-10-09 09:43:57 +01002075 if (ret)
2076 goto err_fence;
Chris Wilsona61007a2016-08-18 17:17:02 +01002077
Chris Wilsona65adaf2017-10-09 09:43:57 +01002078 /* Mark as being mmapped into userspace for later revocation */
2079 assert_rpm_wakelock_held(dev_priv);
2080 if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
2081 list_add(&obj->userfault_link, &dev_priv->mm.userfault_list);
2082 GEM_BUG_ON(!obj->userfault_count);
2083
Chris Wilson71253972017-12-06 12:49:14 +00002084 i915_vma_set_ggtt_write(vma);
2085
Chris Wilsona65adaf2017-10-09 09:43:57 +01002086err_fence:
Chris Wilson3bd40732017-10-09 09:43:56 +01002087 i915_vma_unpin_fence(vma);
Chris Wilsonb8f90962016-08-05 10:14:07 +01002088err_unpin:
Chris Wilson058d88c2016-08-15 10:49:06 +01002089 __i915_vma_unpin(vma);
Chris Wilsonb8f90962016-08-05 10:14:07 +01002090err_unlock:
Jesse Barnesde151cf2008-11-12 10:03:55 -08002091 mutex_unlock(&dev->struct_mutex);
Chris Wilsonb8f90962016-08-05 10:14:07 +01002092err_rpm:
2093 intel_runtime_pm_put(dev_priv);
Chris Wilson40e62d52016-10-28 13:58:41 +01002094 i915_gem_object_unpin_pages(obj);
Chris Wilsonb8f90962016-08-05 10:14:07 +01002095err:
Jesse Barnesde151cf2008-11-12 10:03:55 -08002096 switch (ret) {
Chris Wilsond9bc7e92011-02-07 13:09:31 +00002097 case -EIO:
Daniel Vetter2232f032014-09-04 09:36:18 +02002098 /*
2099 * We eat errors when the gpu is terminally wedged to avoid
2100 * userspace unduly crashing (gl has no provisions for mmaps to
2101 * fail). But any other -EIO isn't ours (e.g. swap in failure)
2102 * and so needs to be reported.
2103 */
2104 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
Paulo Zanonif65c9162013-11-27 18:20:34 -02002105 ret = VM_FAULT_SIGBUS;
2106 break;
2107 }
Chris Wilson045e7692010-11-07 09:18:22 +00002108 case -EAGAIN:
Daniel Vetter571c6082013-09-12 17:57:28 +02002109 /*
2110 * EAGAIN means the gpu is hung and we'll wait for the error
2111 * handler to reset everything when re-faulting in
2112 * i915_mutex_lock_interruptible.
Chris Wilsond9bc7e92011-02-07 13:09:31 +00002113 */
Chris Wilsonc7150892009-09-23 00:43:56 +01002114 case 0:
2115 case -ERESTARTSYS:
Chris Wilsonbed636a2011-02-11 20:31:19 +00002116 case -EINTR:
Dmitry Rogozhkine79e0fe2012-10-03 17:15:26 +03002117 case -EBUSY:
2118 /*
2119 * EBUSY is ok: this just means that another thread
2120 * already did the job.
2121 */
Paulo Zanonif65c9162013-11-27 18:20:34 -02002122 ret = VM_FAULT_NOPAGE;
2123 break;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002124 case -ENOMEM:
Paulo Zanonif65c9162013-11-27 18:20:34 -02002125 ret = VM_FAULT_OOM;
2126 break;
Daniel Vettera7c2e1a2012-10-17 11:17:16 +02002127 case -ENOSPC:
Chris Wilson45d67812014-01-31 11:34:57 +00002128 case -EFAULT:
Paulo Zanonif65c9162013-11-27 18:20:34 -02002129 ret = VM_FAULT_SIGBUS;
2130 break;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002131 default:
Daniel Vettera7c2e1a2012-10-17 11:17:16 +02002132 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
Paulo Zanonif65c9162013-11-27 18:20:34 -02002133 ret = VM_FAULT_SIGBUS;
2134 break;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002135 }
Paulo Zanonif65c9162013-11-27 18:20:34 -02002136 return ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002137}
2138
Chris Wilsona65adaf2017-10-09 09:43:57 +01002139static void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
2140{
2141 struct i915_vma *vma;
2142
2143 GEM_BUG_ON(!obj->userfault_count);
2144
2145 obj->userfault_count = 0;
2146 list_del(&obj->userfault_link);
2147 drm_vma_node_unmap(&obj->base.vma_node,
2148 obj->base.dev->anon_inode->i_mapping);
2149
Chris Wilsone2189dd2017-12-07 21:14:07 +00002150 for_each_ggtt_vma(vma, obj)
Chris Wilsona65adaf2017-10-09 09:43:57 +01002151 i915_vma_unset_userfault(vma);
Chris Wilsona65adaf2017-10-09 09:43:57 +01002152}
2153
Jesse Barnesde151cf2008-11-12 10:03:55 -08002154/**
Chris Wilson901782b2009-07-10 08:18:50 +01002155 * i915_gem_release_mmap - remove physical page mappings
2156 * @obj: obj in question
2157 *
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02002158 * Preserve the reservation of the mmapping with the DRM core code, but
Chris Wilson901782b2009-07-10 08:18:50 +01002159 * relinquish ownership of the pages back to the system.
2160 *
2161 * It is vital that we remove the page mapping if we have mapped a tiled
2162 * object through the GTT and then lose the fence register due to
2163 * resource pressure. Similarly if the object has been moved out of the
2164 * aperture, than pages mapped into userspace must be revoked. Removing the
2165 * mapping will then trigger a page fault on the next user access, allowing
2166 * fixup by i915_gem_fault().
2167 */
Eric Anholtd05ca302009-07-10 13:02:26 -07002168void
Chris Wilson05394f32010-11-08 19:18:58 +00002169i915_gem_release_mmap(struct drm_i915_gem_object *obj)
Chris Wilson901782b2009-07-10 08:18:50 +01002170{
Chris Wilson275f0392016-10-24 13:42:14 +01002171 struct drm_i915_private *i915 = to_i915(obj->base.dev);
Chris Wilson275f0392016-10-24 13:42:14 +01002172
Chris Wilson349f2cc2016-04-13 17:35:12 +01002173 /* Serialisation between user GTT access and our code depends upon
2174 * revoking the CPU's PTE whilst the mutex is held. The next user
2175 * pagefault then has to wait until we release the mutex.
Chris Wilson9c870d02016-10-24 13:42:15 +01002176 *
2177 * Note that RPM complicates somewhat by adding an additional
2178 * requirement that operations to the GGTT be made holding the RPM
2179 * wakeref.
Chris Wilson349f2cc2016-04-13 17:35:12 +01002180 */
Chris Wilson275f0392016-10-24 13:42:14 +01002181 lockdep_assert_held(&i915->drm.struct_mutex);
Chris Wilson9c870d02016-10-24 13:42:15 +01002182 intel_runtime_pm_get(i915);
Chris Wilson349f2cc2016-04-13 17:35:12 +01002183
Chris Wilsona65adaf2017-10-09 09:43:57 +01002184 if (!obj->userfault_count)
Chris Wilson9c870d02016-10-24 13:42:15 +01002185 goto out;
Chris Wilson901782b2009-07-10 08:18:50 +01002186
Chris Wilsona65adaf2017-10-09 09:43:57 +01002187 __i915_gem_object_release_mmap(obj);
Chris Wilson349f2cc2016-04-13 17:35:12 +01002188
2189 /* Ensure that the CPU's PTE are revoked and there are not outstanding
2190 * memory transactions from userspace before we return. The TLB
2191 * flushing implied above by changing the PTE above *should* be
2192 * sufficient, an extra barrier here just provides us with a bit
2193 * of paranoid documentation about our requirement to serialise
2194 * memory writes before touching registers / GSM.
2195 */
2196 wmb();
Chris Wilson9c870d02016-10-24 13:42:15 +01002197
2198out:
2199 intel_runtime_pm_put(i915);
Chris Wilson901782b2009-07-10 08:18:50 +01002200}
2201
Chris Wilson7c108fd2016-10-24 13:42:18 +01002202void i915_gem_runtime_suspend(struct drm_i915_private *dev_priv)
Chris Wilsoneedd10f2014-06-16 08:57:44 +01002203{
Chris Wilson3594a3e2016-10-24 13:42:16 +01002204 struct drm_i915_gem_object *obj, *on;
Chris Wilson7c108fd2016-10-24 13:42:18 +01002205 int i;
Chris Wilsoneedd10f2014-06-16 08:57:44 +01002206
Chris Wilson3594a3e2016-10-24 13:42:16 +01002207 /*
2208 * Only called during RPM suspend. All users of the userfault_list
2209 * must be holding an RPM wakeref to ensure that this can not
2210 * run concurrently with themselves (and use the struct_mutex for
2211 * protection between themselves).
2212 */
2213
2214 list_for_each_entry_safe(obj, on,
Chris Wilsona65adaf2017-10-09 09:43:57 +01002215 &dev_priv->mm.userfault_list, userfault_link)
2216 __i915_gem_object_release_mmap(obj);
Chris Wilson7c108fd2016-10-24 13:42:18 +01002217
2218 /* The fence will be lost when the device powers down. If any were
2219 * in use by hardware (i.e. they are pinned), we should not be powering
2220 * down! All other fences will be reacquired by the user upon waking.
2221 */
2222 for (i = 0; i < dev_priv->num_fence_regs; i++) {
2223 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
2224
Chris Wilsone0ec3ec2017-02-03 12:57:17 +00002225 /* Ideally we want to assert that the fence register is not
2226 * live at this point (i.e. that no piece of code will be
2227 * trying to write through fence + GTT, as that both violates
2228 * our tracking of activity and associated locking/barriers,
2229 * but also is illegal given that the hw is powered down).
2230 *
2231 * Previously we used reg->pin_count as a "liveness" indicator.
2232 * That is not sufficient, and we need a more fine-grained
2233 * tool if we want to have a sanity check here.
2234 */
Chris Wilson7c108fd2016-10-24 13:42:18 +01002235
2236 if (!reg->vma)
2237 continue;
2238
Chris Wilsona65adaf2017-10-09 09:43:57 +01002239 GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
Chris Wilson7c108fd2016-10-24 13:42:18 +01002240 reg->dirty = true;
2241 }
Chris Wilsoneedd10f2014-06-16 08:57:44 +01002242}
2243
Chris Wilsond8cb5082012-08-11 15:41:03 +01002244static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
2245{
Chris Wilsonfac5e232016-07-04 11:34:36 +01002246 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
Chris Wilsonf3f61842016-08-05 10:14:14 +01002247 int err;
Chris Wilsond8cb5082012-08-11 15:41:03 +01002248
Chris Wilsonf3f61842016-08-05 10:14:14 +01002249 err = drm_gem_create_mmap_offset(&obj->base);
Chris Wilsonb42a13d2017-01-06 15:22:40 +00002250 if (likely(!err))
Chris Wilsonf3f61842016-08-05 10:14:14 +01002251 return 0;
Daniel Vetterda494d72012-12-20 15:11:16 +01002252
Chris Wilsonb42a13d2017-01-06 15:22:40 +00002253 /* Attempt to reap some mmap space from dead objects */
2254 do {
2255 err = i915_gem_wait_for_idle(dev_priv, I915_WAIT_INTERRUPTIBLE);
2256 if (err)
2257 break;
Chris Wilsond8cb5082012-08-11 15:41:03 +01002258
Chris Wilsonb42a13d2017-01-06 15:22:40 +00002259 i915_gem_drain_freed_objects(dev_priv);
Chris Wilsonf3f61842016-08-05 10:14:14 +01002260 err = drm_gem_create_mmap_offset(&obj->base);
Chris Wilsonb42a13d2017-01-06 15:22:40 +00002261 if (!err)
2262 break;
2263
2264 } while (flush_delayed_work(&dev_priv->gt.retire_work));
Daniel Vetterda494d72012-12-20 15:11:16 +01002265
Chris Wilsonf3f61842016-08-05 10:14:14 +01002266 return err;
Chris Wilsond8cb5082012-08-11 15:41:03 +01002267}
2268
2269static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2270{
Chris Wilsond8cb5082012-08-11 15:41:03 +01002271 drm_gem_free_mmap_offset(&obj->base);
2272}
2273
Dave Airlieda6b51d2014-12-24 13:11:17 +10002274int
Dave Airlieff72145b2011-02-07 12:16:14 +10002275i915_gem_mmap_gtt(struct drm_file *file,
2276 struct drm_device *dev,
Dave Airlieda6b51d2014-12-24 13:11:17 +10002277 uint32_t handle,
Dave Airlieff72145b2011-02-07 12:16:14 +10002278 uint64_t *offset)
Jesse Barnesde151cf2008-11-12 10:03:55 -08002279{
Chris Wilson05394f32010-11-08 19:18:58 +00002280 struct drm_i915_gem_object *obj;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002281 int ret;
2282
Chris Wilson03ac0642016-07-20 13:31:51 +01002283 obj = i915_gem_object_lookup(file, handle);
Chris Wilsonf3f61842016-08-05 10:14:14 +01002284 if (!obj)
2285 return -ENOENT;
Chris Wilsonab182822009-09-22 18:46:17 +01002286
Chris Wilsond8cb5082012-08-11 15:41:03 +01002287 ret = i915_gem_object_create_mmap_offset(obj);
Chris Wilsonf3f61842016-08-05 10:14:14 +01002288 if (ret == 0)
2289 *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
Jesse Barnesde151cf2008-11-12 10:03:55 -08002290
Chris Wilsonf0cd5182016-10-28 13:58:43 +01002291 i915_gem_object_put(obj);
Chris Wilson1d7cfea2010-10-17 09:45:41 +01002292 return ret;
Jesse Barnesde151cf2008-11-12 10:03:55 -08002293}
2294
Dave Airlieff72145b2011-02-07 12:16:14 +10002295/**
2296 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
2297 * @dev: DRM device
2298 * @data: GTT mapping ioctl data
2299 * @file: GEM object info
2300 *
2301 * Simply returns the fake offset to userspace so it can mmap it.
2302 * The mmap call will end up in drm_gem_mmap(), which will set things
2303 * up so we can get faults in the handler above.
2304 *
2305 * The fault handler will take care of binding the object into the GTT
2306 * (since it may have been evicted to make room for something), allocating
2307 * a fence register, and mapping the appropriate aperture address into
2308 * userspace.
2309 */
2310int
2311i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2312 struct drm_file *file)
2313{
2314 struct drm_i915_gem_mmap_gtt *args = data;
2315
Dave Airlieda6b51d2014-12-24 13:11:17 +10002316 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
Dave Airlieff72145b2011-02-07 12:16:14 +10002317}
2318
Daniel Vetter225067e2012-08-20 10:23:20 +02002319/* Immediately discard the backing storage */
2320static void
2321i915_gem_object_truncate(struct drm_i915_gem_object *obj)
Chris Wilsone5281cc2010-10-28 13:45:36 +01002322{
Chris Wilson4d6294bf2012-08-11 15:41:05 +01002323 i915_gem_object_free_mmap_offset(obj);
Daniel Vetter1286ff72012-05-10 15:25:09 +02002324
Chris Wilson4d6294bf2012-08-11 15:41:05 +01002325 if (obj->base.filp == NULL)
2326 return;
2327
Daniel Vetter225067e2012-08-20 10:23:20 +02002328 /* Our goal here is to return as much of the memory as
2329 * is possible back to the system as we are called from OOM.
2330 * To do this we must instruct the shmfs to drop all of its
2331 * backing pages, *now*.
Chris Wilsone5281cc2010-10-28 13:45:36 +01002332 */
Chris Wilson55372522014-03-25 13:23:06 +00002333 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
Chris Wilsona4f5ea62016-10-28 13:58:35 +01002334 obj->mm.madv = __I915_MADV_PURGED;
Chris Wilson4e5462e2017-03-07 13:20:31 +00002335 obj->mm.pages = ERR_PTR(-EFAULT);
Chris Wilsone5281cc2010-10-28 13:45:36 +01002336}
Chris Wilsone5281cc2010-10-28 13:45:36 +01002337
Chris Wilson55372522014-03-25 13:23:06 +00002338/* Try to discard unwanted pages */
Chris Wilson03ac84f2016-10-28 13:58:36 +01002339void __i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
Daniel Vetter225067e2012-08-20 10:23:20 +02002340{
Chris Wilson55372522014-03-25 13:23:06 +00002341 struct address_space *mapping;
2342
Chris Wilson1233e2d2016-10-28 13:58:37 +01002343 lockdep_assert_held(&obj->mm.lock);
Chris Wilsonf1fa4f42017-10-13 21:26:13 +01002344 GEM_BUG_ON(i915_gem_object_has_pages(obj));
Chris Wilson1233e2d2016-10-28 13:58:37 +01002345
Chris Wilsona4f5ea62016-10-28 13:58:35 +01002346 switch (obj->mm.madv) {
Chris Wilson55372522014-03-25 13:23:06 +00002347 case I915_MADV_DONTNEED:
2348 i915_gem_object_truncate(obj);
2349 case __I915_MADV_PURGED:
2350 return;
2351 }
2352
2353 if (obj->base.filp == NULL)
2354 return;
2355
Al Viro93c76a32015-12-04 23:45:44 -05002356 mapping = obj->base.filp->f_mapping,
Chris Wilson55372522014-03-25 13:23:06 +00002357 invalidate_mapping_pages(mapping, 0, (loff_t)-1);
Chris Wilsone5281cc2010-10-28 13:45:36 +01002358}
2359
Chris Wilson5cdf5882010-09-27 15:51:07 +01002360static void
Chris Wilson03ac84f2016-10-28 13:58:36 +01002361i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj,
2362 struct sg_table *pages)
Eric Anholt673a3942008-07-30 12:06:12 -07002363{
Dave Gordon85d12252016-05-20 11:54:06 +01002364 struct sgt_iter sgt_iter;
2365 struct page *page;
Daniel Vetter1286ff72012-05-10 15:25:09 +02002366
Chris Wilsone5facdf2016-12-23 14:57:57 +00002367 __i915_gem_object_release_shmem(obj, pages, true);
Eric Anholt856fa192009-03-19 14:10:50 -07002368
Chris Wilson03ac84f2016-10-28 13:58:36 +01002369 i915_gem_gtt_finish_pages(obj, pages);
Imre Deake2273302015-07-09 12:59:05 +03002370
Daniel Vetter6dacfd22011-09-12 21:30:02 +02002371 if (i915_gem_object_needs_bit17_swizzle(obj))
Chris Wilson03ac84f2016-10-28 13:58:36 +01002372 i915_gem_object_save_bit_17_swizzle(obj, pages);
Eric Anholt280b7132009-03-12 16:56:27 -07002373
Chris Wilson03ac84f2016-10-28 13:58:36 +01002374 for_each_sgt_page(page, sgt_iter, pages) {
Chris Wilsona4f5ea62016-10-28 13:58:35 +01002375 if (obj->mm.dirty)
Chris Wilson9da3da62012-06-01 15:20:22 +01002376 set_page_dirty(page);
Chris Wilson3ef94da2009-09-14 16:50:29 +01002377
Chris Wilsona4f5ea62016-10-28 13:58:35 +01002378 if (obj->mm.madv == I915_MADV_WILLNEED)
Chris Wilson9da3da62012-06-01 15:20:22 +01002379 mark_page_accessed(page);
Chris Wilson3ef94da2009-09-14 16:50:29 +01002380
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03002381 put_page(page);
Chris Wilson3ef94da2009-09-14 16:50:29 +01002382 }
Chris Wilsona4f5ea62016-10-28 13:58:35 +01002383 obj->mm.dirty = false;
Eric Anholt673a3942008-07-30 12:06:12 -07002384
Chris Wilson03ac84f2016-10-28 13:58:36 +01002385 sg_free_table(pages);
2386 kfree(pages);
Chris Wilson37e680a2012-06-07 15:38:42 +01002387}
2388
Chris Wilson96d77632016-10-28 13:58:33 +01002389static void __i915_gem_object_reset_page_iter(struct drm_i915_gem_object *obj)
2390{
2391 struct radix_tree_iter iter;
Ville Syrjäläc23aa712017-09-01 20:12:51 +03002392 void __rcu **slot;
Chris Wilson96d77632016-10-28 13:58:33 +01002393
Chris Wilsonbea6e982017-10-26 14:00:31 +01002394 rcu_read_lock();
Chris Wilsona4f5ea62016-10-28 13:58:35 +01002395 radix_tree_for_each_slot(slot, &obj->mm.get_page.radix, &iter, 0)
2396 radix_tree_delete(&obj->mm.get_page.radix, iter.index);
Chris Wilsonbea6e982017-10-26 14:00:31 +01002397 rcu_read_unlock();
Chris Wilson96d77632016-10-28 13:58:33 +01002398}
2399
Chris Wilson548625e2016-11-01 12:11:34 +00002400void __i915_gem_object_put_pages(struct drm_i915_gem_object *obj,
2401 enum i915_mm_subclass subclass)
Chris Wilson37e680a2012-06-07 15:38:42 +01002402{
Chris Wilsonf2123812017-10-16 12:40:37 +01002403 struct drm_i915_private *i915 = to_i915(obj->base.dev);
Chris Wilson03ac84f2016-10-28 13:58:36 +01002404 struct sg_table *pages;
Chris Wilson37e680a2012-06-07 15:38:42 +01002405
Chris Wilsona4f5ea62016-10-28 13:58:35 +01002406 if (i915_gem_object_has_pinned_pages(obj))
Chris Wilson03ac84f2016-10-28 13:58:36 +01002407 return;
Chris Wilsona5570172012-09-04 21:02:54 +01002408
Chris Wilson15717de2016-08-04 07:52:26 +01002409 GEM_BUG_ON(obj->bind_count);
Chris Wilsonf1fa4f42017-10-13 21:26:13 +01002410 if (!i915_gem_object_has_pages(obj))
Chris Wilson1233e2d2016-10-28 13:58:37 +01002411 return;
2412
2413 /* May be called by shrinker from within get_pages() (on another bo) */
Chris Wilson548625e2016-11-01 12:11:34 +00002414 mutex_lock_nested(&obj->mm.lock, subclass);
Chris Wilson1233e2d2016-10-28 13:58:37 +01002415 if (unlikely(atomic_read(&obj->mm.pages_pin_count)))
2416 goto unlock;
Ben Widawsky3e123022013-07-31 17:00:04 -07002417
Chris Wilsona2165e32012-12-03 11:49:00 +00002418 /* ->put_pages might need to allocate memory for the bit17 swizzle
2419 * array, hence protect them from being reaped by removing them from gtt
2420 * lists early. */
Chris Wilson03ac84f2016-10-28 13:58:36 +01002421 pages = fetch_and_zero(&obj->mm.pages);
2422 GEM_BUG_ON(!pages);
Chris Wilsona2165e32012-12-03 11:49:00 +00002423
Chris Wilsonf2123812017-10-16 12:40:37 +01002424 spin_lock(&i915->mm.obj_lock);
2425 list_del(&obj->mm.link);
2426 spin_unlock(&i915->mm.obj_lock);
2427
Chris Wilsona4f5ea62016-10-28 13:58:35 +01002428 if (obj->mm.mapping) {
Chris Wilson4b30cb22016-08-18 17:16:42 +01002429 void *ptr;
2430
Chris Wilson0ce81782017-05-17 13:09:59 +01002431 ptr = page_mask_bits(obj->mm.mapping);
Chris Wilson4b30cb22016-08-18 17:16:42 +01002432 if (is_vmalloc_addr(ptr))
2433 vunmap(ptr);
Chris Wilsonfb8621d2016-04-08 12:11:14 +01002434 else
Chris Wilson4b30cb22016-08-18 17:16:42 +01002435 kunmap(kmap_to_page(ptr));
2436
Chris Wilsona4f5ea62016-10-28 13:58:35 +01002437 obj->mm.mapping = NULL;
Chris Wilson0a798eb2016-04-08 12:11:11 +01002438 }
2439
Chris Wilson96d77632016-10-28 13:58:33 +01002440 __i915_gem_object_reset_page_iter(obj);
2441
Chris Wilson4e5462e2017-03-07 13:20:31 +00002442 if (!IS_ERR(pages))
2443 obj->ops->put_pages(obj, pages);
2444
Matthew Aulda5c081662017-10-06 23:18:18 +01002445 obj->mm.page_sizes.phys = obj->mm.page_sizes.sg = 0;
2446
Chris Wilson1233e2d2016-10-28 13:58:37 +01002447unlock:
2448 mutex_unlock(&obj->mm.lock);
Chris Wilson6c085a72012-08-20 11:40:46 +02002449}
2450
Chris Wilson935a2f72017-02-13 17:15:13 +00002451static bool i915_sg_trim(struct sg_table *orig_st)
Tvrtko Ursulin0c40ce12016-11-09 15:13:43 +00002452{
2453 struct sg_table new_st;
2454 struct scatterlist *sg, *new_sg;
2455 unsigned int i;
2456
2457 if (orig_st->nents == orig_st->orig_nents)
Chris Wilson935a2f72017-02-13 17:15:13 +00002458 return false;
Tvrtko Ursulin0c40ce12016-11-09 15:13:43 +00002459
Chris Wilson8bfc478f2016-12-23 14:57:58 +00002460 if (sg_alloc_table(&new_st, orig_st->nents, GFP_KERNEL | __GFP_NOWARN))
Chris Wilson935a2f72017-02-13 17:15:13 +00002461 return false;
Tvrtko Ursulin0c40ce12016-11-09 15:13:43 +00002462
2463 new_sg = new_st.sgl;
2464 for_each_sg(orig_st->sgl, sg, orig_st->nents, i) {
2465 sg_set_page(new_sg, sg_page(sg), sg->length, 0);
2466 /* called before being DMA mapped, no need to copy sg->dma_* */
2467 new_sg = sg_next(new_sg);
2468 }
Chris Wilsonc2dc6cc2016-12-19 12:43:46 +00002469 GEM_BUG_ON(new_sg); /* Should walk exactly nents and hit the end */
Tvrtko Ursulin0c40ce12016-11-09 15:13:43 +00002470
2471 sg_free_table(orig_st);
2472
2473 *orig_st = new_st;
Chris Wilson935a2f72017-02-13 17:15:13 +00002474 return true;
Tvrtko Ursulin0c40ce12016-11-09 15:13:43 +00002475}
2476
Matthew Auldb91b09e2017-10-06 23:18:17 +01002477static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
Eric Anholt673a3942008-07-30 12:06:12 -07002478{
Chris Wilsonfac5e232016-07-04 11:34:36 +01002479 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
Chris Wilsond766ef52016-12-19 12:43:45 +00002480 const unsigned long page_count = obj->base.size / PAGE_SIZE;
2481 unsigned long i;
Eric Anholt673a3942008-07-30 12:06:12 -07002482 struct address_space *mapping;
Chris Wilson9da3da62012-06-01 15:20:22 +01002483 struct sg_table *st;
2484 struct scatterlist *sg;
Dave Gordon85d12252016-05-20 11:54:06 +01002485 struct sgt_iter sgt_iter;
Eric Anholt673a3942008-07-30 12:06:12 -07002486 struct page *page;
Imre Deak90797e62013-02-18 19:28:03 +02002487 unsigned long last_pfn = 0; /* suppress gcc warning */
Tvrtko Ursulin56024522017-08-03 10:14:17 +01002488 unsigned int max_segment = i915_sg_segment_size();
Matthew Auld84e89782017-10-09 12:00:24 +01002489 unsigned int sg_page_sizes;
Chris Wilson4846bf02017-06-09 12:03:46 +01002490 gfp_t noreclaim;
Imre Deake2273302015-07-09 12:59:05 +03002491 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07002492
Chris Wilson6c085a72012-08-20 11:40:46 +02002493 /* Assert that the object is not currently in any GPU domain. As it
2494 * wasn't in the GTT, there shouldn't be any way it could have been in
2495 * a GPU cache
2496 */
Christian Königc0a51fd2018-02-16 13:43:38 +01002497 GEM_BUG_ON(obj->read_domains & I915_GEM_GPU_DOMAINS);
2498 GEM_BUG_ON(obj->write_domain & I915_GEM_GPU_DOMAINS);
Chris Wilson6c085a72012-08-20 11:40:46 +02002499
Chris Wilson9da3da62012-06-01 15:20:22 +01002500 st = kmalloc(sizeof(*st), GFP_KERNEL);
2501 if (st == NULL)
Matthew Auldb91b09e2017-10-06 23:18:17 +01002502 return -ENOMEM;
Eric Anholt673a3942008-07-30 12:06:12 -07002503
Chris Wilsond766ef52016-12-19 12:43:45 +00002504rebuild_st:
Chris Wilson9da3da62012-06-01 15:20:22 +01002505 if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
Chris Wilson9da3da62012-06-01 15:20:22 +01002506 kfree(st);
Matthew Auldb91b09e2017-10-06 23:18:17 +01002507 return -ENOMEM;
Chris Wilson9da3da62012-06-01 15:20:22 +01002508 }
2509
2510 /* Get the list of pages out of our struct file. They'll be pinned
2511 * at this point until we release them.
2512 *
2513 * Fail silently without starting the shrinker
2514 */
Al Viro93c76a32015-12-04 23:45:44 -05002515 mapping = obj->base.filp->f_mapping;
Chris Wilson0f6ab552017-06-09 12:03:48 +01002516 noreclaim = mapping_gfp_constraint(mapping, ~__GFP_RECLAIM);
Chris Wilson4846bf02017-06-09 12:03:46 +01002517 noreclaim |= __GFP_NORETRY | __GFP_NOWARN;
2518
Imre Deak90797e62013-02-18 19:28:03 +02002519 sg = st->sgl;
2520 st->nents = 0;
Matthew Auld84e89782017-10-09 12:00:24 +01002521 sg_page_sizes = 0;
Imre Deak90797e62013-02-18 19:28:03 +02002522 for (i = 0; i < page_count; i++) {
Chris Wilson4846bf02017-06-09 12:03:46 +01002523 const unsigned int shrink[] = {
2524 I915_SHRINK_BOUND | I915_SHRINK_UNBOUND | I915_SHRINK_PURGEABLE,
2525 0,
2526 }, *s = shrink;
2527 gfp_t gfp = noreclaim;
2528
2529 do {
Chris Wilson6c085a72012-08-20 11:40:46 +02002530 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
Chris Wilson4846bf02017-06-09 12:03:46 +01002531 if (likely(!IS_ERR(page)))
2532 break;
2533
2534 if (!*s) {
2535 ret = PTR_ERR(page);
2536 goto err_sg;
2537 }
2538
Chris Wilson912d5722017-09-06 16:19:30 -07002539 i915_gem_shrink(dev_priv, 2 * page_count, NULL, *s++);
Chris Wilson4846bf02017-06-09 12:03:46 +01002540 cond_resched();
Chris Wilson24f8e002017-03-22 11:05:21 +00002541
Chris Wilson6c085a72012-08-20 11:40:46 +02002542 /* We've tried hard to allocate the memory by reaping
2543 * our own buffer, now let the real VM do its job and
2544 * go down in flames if truly OOM.
Chris Wilson24f8e002017-03-22 11:05:21 +00002545 *
2546 * However, since graphics tend to be disposable,
2547 * defer the oom here by reporting the ENOMEM back
2548 * to userspace.
Chris Wilson6c085a72012-08-20 11:40:46 +02002549 */
Chris Wilson4846bf02017-06-09 12:03:46 +01002550 if (!*s) {
2551 /* reclaim and warn, but no oom */
2552 gfp = mapping_gfp_mask(mapping);
Chris Wilsoneaf41802017-06-09 12:03:47 +01002553
2554 /* Our bo are always dirty and so we require
2555 * kswapd to reclaim our pages (direct reclaim
2556 * does not effectively begin pageout of our
2557 * buffers on its own). However, direct reclaim
2558 * only waits for kswapd when under allocation
2559 * congestion. So as a result __GFP_RECLAIM is
2560 * unreliable and fails to actually reclaim our
2561 * dirty pages -- unless you try over and over
2562 * again with !__GFP_NORETRY. However, we still
2563 * want to fail this allocation rather than
2564 * trigger the out-of-memory killer and for
Michal Hockodbb32952017-07-12 14:36:55 -07002565 * this we want __GFP_RETRY_MAYFAIL.
Chris Wilsoneaf41802017-06-09 12:03:47 +01002566 */
Michal Hockodbb32952017-07-12 14:36:55 -07002567 gfp |= __GFP_RETRY_MAYFAIL;
Imre Deake2273302015-07-09 12:59:05 +03002568 }
Chris Wilson4846bf02017-06-09 12:03:46 +01002569 } while (1);
2570
Chris Wilson871dfbd2016-10-11 09:20:21 +01002571 if (!i ||
2572 sg->length >= max_segment ||
2573 page_to_pfn(page) != last_pfn + 1) {
Matthew Aulda5c081662017-10-06 23:18:18 +01002574 if (i) {
Matthew Auld84e89782017-10-09 12:00:24 +01002575 sg_page_sizes |= sg->length;
Imre Deak90797e62013-02-18 19:28:03 +02002576 sg = sg_next(sg);
Matthew Aulda5c081662017-10-06 23:18:18 +01002577 }
Imre Deak90797e62013-02-18 19:28:03 +02002578 st->nents++;
2579 sg_set_page(sg, page, PAGE_SIZE, 0);
2580 } else {
2581 sg->length += PAGE_SIZE;
2582 }
2583 last_pfn = page_to_pfn(page);
Daniel Vetter3bbbe702013-10-07 17:15:45 -03002584
2585 /* Check that the i965g/gm workaround works. */
2586 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
Eric Anholt673a3942008-07-30 12:06:12 -07002587 }
Matthew Aulda5c081662017-10-06 23:18:18 +01002588 if (sg) { /* loop terminated early; short sg table */
Matthew Auld84e89782017-10-09 12:00:24 +01002589 sg_page_sizes |= sg->length;
Konrad Rzeszutek Wilk426729d2013-06-24 11:47:48 -04002590 sg_mark_end(sg);
Matthew Aulda5c081662017-10-06 23:18:18 +01002591 }
Chris Wilson74ce6b62012-10-19 15:51:06 +01002592
Tvrtko Ursulin0c40ce12016-11-09 15:13:43 +00002593 /* Trim unused sg entries to avoid wasting memory. */
2594 i915_sg_trim(st);
2595
Chris Wilson03ac84f2016-10-28 13:58:36 +01002596 ret = i915_gem_gtt_prepare_pages(obj, st);
Chris Wilsond766ef52016-12-19 12:43:45 +00002597 if (ret) {
2598 /* DMA remapping failed? One possible cause is that
2599 * it could not reserve enough large entries, asking
2600 * for PAGE_SIZE chunks instead may be helpful.
2601 */
2602 if (max_segment > PAGE_SIZE) {
2603 for_each_sgt_page(page, sgt_iter, st)
2604 put_page(page);
2605 sg_free_table(st);
2606
2607 max_segment = PAGE_SIZE;
2608 goto rebuild_st;
2609 } else {
2610 dev_warn(&dev_priv->drm.pdev->dev,
2611 "Failed to DMA remap %lu pages\n",
2612 page_count);
2613 goto err_pages;
2614 }
2615 }
Imre Deake2273302015-07-09 12:59:05 +03002616
Eric Anholt673a3942008-07-30 12:06:12 -07002617 if (i915_gem_object_needs_bit17_swizzle(obj))
Chris Wilson03ac84f2016-10-28 13:58:36 +01002618 i915_gem_object_do_bit_17_swizzle(obj, st);
Eric Anholt673a3942008-07-30 12:06:12 -07002619
Matthew Auld84e89782017-10-09 12:00:24 +01002620 __i915_gem_object_set_pages(obj, st, sg_page_sizes);
Matthew Auldb91b09e2017-10-06 23:18:17 +01002621
2622 return 0;
Eric Anholt673a3942008-07-30 12:06:12 -07002623
Chris Wilsonb17993b2016-11-14 11:29:30 +00002624err_sg:
Imre Deak90797e62013-02-18 19:28:03 +02002625 sg_mark_end(sg);
Chris Wilsonb17993b2016-11-14 11:29:30 +00002626err_pages:
Dave Gordon85d12252016-05-20 11:54:06 +01002627 for_each_sgt_page(page, sgt_iter, st)
2628 put_page(page);
Chris Wilson9da3da62012-06-01 15:20:22 +01002629 sg_free_table(st);
2630 kfree(st);
Chris Wilson0820baf2014-03-25 13:23:03 +00002631
2632 /* shmemfs first checks if there is enough memory to allocate the page
2633 * and reports ENOSPC should there be insufficient, along with the usual
2634 * ENOMEM for a genuine allocation failure.
2635 *
2636 * We use ENOSPC in our driver to mean that we have run out of aperture
2637 * space and so want to translate the error from shmemfs back to our
2638 * usual understanding of ENOMEM.
2639 */
Imre Deake2273302015-07-09 12:59:05 +03002640 if (ret == -ENOSPC)
2641 ret = -ENOMEM;
2642
Matthew Auldb91b09e2017-10-06 23:18:17 +01002643 return ret;
Chris Wilson03ac84f2016-10-28 13:58:36 +01002644}
2645
2646void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj,
Matthew Aulda5c081662017-10-06 23:18:18 +01002647 struct sg_table *pages,
Matthew Auld84e89782017-10-09 12:00:24 +01002648 unsigned int sg_page_sizes)
Chris Wilson03ac84f2016-10-28 13:58:36 +01002649{
Matthew Aulda5c081662017-10-06 23:18:18 +01002650 struct drm_i915_private *i915 = to_i915(obj->base.dev);
2651 unsigned long supported = INTEL_INFO(i915)->page_sizes;
2652 int i;
2653
Chris Wilson1233e2d2016-10-28 13:58:37 +01002654 lockdep_assert_held(&obj->mm.lock);
Chris Wilson03ac84f2016-10-28 13:58:36 +01002655
2656 obj->mm.get_page.sg_pos = pages->sgl;
2657 obj->mm.get_page.sg_idx = 0;
2658
2659 obj->mm.pages = pages;
Chris Wilson2c3a3f42016-11-04 10:30:01 +00002660
2661 if (i915_gem_object_is_tiled(obj) &&
Chris Wilsonf2123812017-10-16 12:40:37 +01002662 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
Chris Wilson2c3a3f42016-11-04 10:30:01 +00002663 GEM_BUG_ON(obj->mm.quirked);
2664 __i915_gem_object_pin_pages(obj);
2665 obj->mm.quirked = true;
2666 }
Matthew Aulda5c081662017-10-06 23:18:18 +01002667
Matthew Auld84e89782017-10-09 12:00:24 +01002668 GEM_BUG_ON(!sg_page_sizes);
2669 obj->mm.page_sizes.phys = sg_page_sizes;
Matthew Aulda5c081662017-10-06 23:18:18 +01002670
2671 /*
Matthew Auld84e89782017-10-09 12:00:24 +01002672 * Calculate the supported page-sizes which fit into the given
2673 * sg_page_sizes. This will give us the page-sizes which we may be able
2674 * to use opportunistically when later inserting into the GTT. For
2675 * example if phys=2G, then in theory we should be able to use 1G, 2M,
2676 * 64K or 4K pages, although in practice this will depend on a number of
2677 * other factors.
Matthew Aulda5c081662017-10-06 23:18:18 +01002678 */
2679 obj->mm.page_sizes.sg = 0;
2680 for_each_set_bit(i, &supported, ilog2(I915_GTT_MAX_PAGE_SIZE) + 1) {
2681 if (obj->mm.page_sizes.phys & ~0u << i)
2682 obj->mm.page_sizes.sg |= BIT(i);
2683 }
Matthew Aulda5c081662017-10-06 23:18:18 +01002684 GEM_BUG_ON(!HAS_PAGE_SIZES(i915, obj->mm.page_sizes.sg));
Chris Wilsonf2123812017-10-16 12:40:37 +01002685
2686 spin_lock(&i915->mm.obj_lock);
2687 list_add(&obj->mm.link, &i915->mm.unbound_list);
2688 spin_unlock(&i915->mm.obj_lock);
Chris Wilson03ac84f2016-10-28 13:58:36 +01002689}
2690
2691static int ____i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2692{
Matthew Auldb91b09e2017-10-06 23:18:17 +01002693 int err;
Chris Wilson03ac84f2016-10-28 13:58:36 +01002694
2695 if (unlikely(obj->mm.madv != I915_MADV_WILLNEED)) {
2696 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2697 return -EFAULT;
2698 }
2699
Matthew Auldb91b09e2017-10-06 23:18:17 +01002700 err = obj->ops->get_pages(obj);
Matthew Auldb65a9b92017-12-18 10:38:55 +00002701 GEM_BUG_ON(!err && !i915_gem_object_has_pages(obj));
Chris Wilson03ac84f2016-10-28 13:58:36 +01002702
Matthew Auldb91b09e2017-10-06 23:18:17 +01002703 return err;
Eric Anholt673a3942008-07-30 12:06:12 -07002704}
2705
Chris Wilson37e680a2012-06-07 15:38:42 +01002706/* Ensure that the associated pages are gathered from the backing storage
Chris Wilson1233e2d2016-10-28 13:58:37 +01002707 * and pinned into our object. i915_gem_object_pin_pages() may be called
Chris Wilson37e680a2012-06-07 15:38:42 +01002708 * multiple times before they are released by a single call to
Chris Wilson1233e2d2016-10-28 13:58:37 +01002709 * i915_gem_object_unpin_pages() - once the pages are no longer referenced
Chris Wilson37e680a2012-06-07 15:38:42 +01002710 * either as a result of memory pressure (reaping pages under the shrinker)
2711 * or as the object is itself released.
2712 */
Chris Wilsona4f5ea62016-10-28 13:58:35 +01002713int __i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
Chris Wilson37e680a2012-06-07 15:38:42 +01002714{
Chris Wilson03ac84f2016-10-28 13:58:36 +01002715 int err;
Chris Wilson37e680a2012-06-07 15:38:42 +01002716
Chris Wilson1233e2d2016-10-28 13:58:37 +01002717 err = mutex_lock_interruptible(&obj->mm.lock);
2718 if (err)
2719 return err;
Chris Wilson4c7d62c2016-10-28 13:58:32 +01002720
Chris Wilsonf1fa4f42017-10-13 21:26:13 +01002721 if (unlikely(!i915_gem_object_has_pages(obj))) {
Chris Wilson88c880b2017-09-06 14:52:20 +01002722 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
2723
Chris Wilson2c3a3f42016-11-04 10:30:01 +00002724 err = ____i915_gem_object_get_pages(obj);
2725 if (err)
2726 goto unlock;
2727
2728 smp_mb__before_atomic();
Chris Wilson1233e2d2016-10-28 13:58:37 +01002729 }
Chris Wilson2c3a3f42016-11-04 10:30:01 +00002730 atomic_inc(&obj->mm.pages_pin_count);
Chris Wilson43e28f02013-01-08 10:53:09 +00002731
Chris Wilson1233e2d2016-10-28 13:58:37 +01002732unlock:
2733 mutex_unlock(&obj->mm.lock);
Chris Wilson03ac84f2016-10-28 13:58:36 +01002734 return err;
Eric Anholt673a3942008-07-30 12:06:12 -07002735}
2736
Dave Gordondd6034c2016-05-20 11:54:04 +01002737/* The 'mapping' part of i915_gem_object_pin_map() below */
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002738static void *i915_gem_object_map(const struct drm_i915_gem_object *obj,
2739 enum i915_map_type type)
Dave Gordondd6034c2016-05-20 11:54:04 +01002740{
2741 unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
Chris Wilsona4f5ea62016-10-28 13:58:35 +01002742 struct sg_table *sgt = obj->mm.pages;
Dave Gordon85d12252016-05-20 11:54:06 +01002743 struct sgt_iter sgt_iter;
2744 struct page *page;
Dave Gordonb338fa42016-05-20 11:54:05 +01002745 struct page *stack_pages[32];
2746 struct page **pages = stack_pages;
Dave Gordondd6034c2016-05-20 11:54:04 +01002747 unsigned long i = 0;
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002748 pgprot_t pgprot;
Dave Gordondd6034c2016-05-20 11:54:04 +01002749 void *addr;
2750
2751 /* A single page can always be kmapped */
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002752 if (n_pages == 1 && type == I915_MAP_WB)
Dave Gordondd6034c2016-05-20 11:54:04 +01002753 return kmap(sg_page(sgt->sgl));
2754
Dave Gordonb338fa42016-05-20 11:54:05 +01002755 if (n_pages > ARRAY_SIZE(stack_pages)) {
2756 /* Too big for stack -- allocate temporary array instead */
Michal Hocko0ee931c2017-09-13 16:28:29 -07002757 pages = kvmalloc_array(n_pages, sizeof(*pages), GFP_KERNEL);
Dave Gordonb338fa42016-05-20 11:54:05 +01002758 if (!pages)
2759 return NULL;
2760 }
Dave Gordondd6034c2016-05-20 11:54:04 +01002761
Dave Gordon85d12252016-05-20 11:54:06 +01002762 for_each_sgt_page(page, sgt_iter, sgt)
2763 pages[i++] = page;
Dave Gordondd6034c2016-05-20 11:54:04 +01002764
2765 /* Check that we have the expected number of pages */
2766 GEM_BUG_ON(i != n_pages);
2767
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002768 switch (type) {
Chris Wilsona575c672017-08-28 11:46:31 +01002769 default:
2770 MISSING_CASE(type);
2771 /* fallthrough to use PAGE_KERNEL anyway */
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002772 case I915_MAP_WB:
2773 pgprot = PAGE_KERNEL;
2774 break;
2775 case I915_MAP_WC:
2776 pgprot = pgprot_writecombine(PAGE_KERNEL_IO);
2777 break;
2778 }
2779 addr = vmap(pages, n_pages, 0, pgprot);
Dave Gordondd6034c2016-05-20 11:54:04 +01002780
Dave Gordonb338fa42016-05-20 11:54:05 +01002781 if (pages != stack_pages)
Michal Hocko20981052017-05-17 14:23:12 +02002782 kvfree(pages);
Dave Gordondd6034c2016-05-20 11:54:04 +01002783
2784 return addr;
2785}
2786
2787/* get, pin, and map the pages of the object into kernel space */
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002788void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
2789 enum i915_map_type type)
Chris Wilson0a798eb2016-04-08 12:11:11 +01002790{
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002791 enum i915_map_type has_type;
2792 bool pinned;
2793 void *ptr;
Chris Wilson0a798eb2016-04-08 12:11:11 +01002794 int ret;
2795
Tina Zhanga03f3952017-11-14 10:25:13 +00002796 if (unlikely(!i915_gem_object_has_struct_page(obj)))
2797 return ERR_PTR(-ENXIO);
Chris Wilson0a798eb2016-04-08 12:11:11 +01002798
Chris Wilson1233e2d2016-10-28 13:58:37 +01002799 ret = mutex_lock_interruptible(&obj->mm.lock);
Chris Wilson0a798eb2016-04-08 12:11:11 +01002800 if (ret)
2801 return ERR_PTR(ret);
2802
Chris Wilsona575c672017-08-28 11:46:31 +01002803 pinned = !(type & I915_MAP_OVERRIDE);
2804 type &= ~I915_MAP_OVERRIDE;
2805
Chris Wilson1233e2d2016-10-28 13:58:37 +01002806 if (!atomic_inc_not_zero(&obj->mm.pages_pin_count)) {
Chris Wilsonf1fa4f42017-10-13 21:26:13 +01002807 if (unlikely(!i915_gem_object_has_pages(obj))) {
Chris Wilson88c880b2017-09-06 14:52:20 +01002808 GEM_BUG_ON(i915_gem_object_has_pinned_pages(obj));
2809
Chris Wilson2c3a3f42016-11-04 10:30:01 +00002810 ret = ____i915_gem_object_get_pages(obj);
2811 if (ret)
2812 goto err_unlock;
Chris Wilson1233e2d2016-10-28 13:58:37 +01002813
Chris Wilson2c3a3f42016-11-04 10:30:01 +00002814 smp_mb__before_atomic();
2815 }
2816 atomic_inc(&obj->mm.pages_pin_count);
Chris Wilson1233e2d2016-10-28 13:58:37 +01002817 pinned = false;
2818 }
Chris Wilsonf1fa4f42017-10-13 21:26:13 +01002819 GEM_BUG_ON(!i915_gem_object_has_pages(obj));
Chris Wilson0a798eb2016-04-08 12:11:11 +01002820
Chris Wilson0ce81782017-05-17 13:09:59 +01002821 ptr = page_unpack_bits(obj->mm.mapping, &has_type);
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002822 if (ptr && has_type != type) {
2823 if (pinned) {
2824 ret = -EBUSY;
Chris Wilson1233e2d2016-10-28 13:58:37 +01002825 goto err_unpin;
Chris Wilson0a798eb2016-04-08 12:11:11 +01002826 }
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002827
2828 if (is_vmalloc_addr(ptr))
2829 vunmap(ptr);
2830 else
2831 kunmap(kmap_to_page(ptr));
2832
Chris Wilsona4f5ea62016-10-28 13:58:35 +01002833 ptr = obj->mm.mapping = NULL;
Chris Wilson0a798eb2016-04-08 12:11:11 +01002834 }
2835
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002836 if (!ptr) {
2837 ptr = i915_gem_object_map(obj, type);
2838 if (!ptr) {
2839 ret = -ENOMEM;
Chris Wilson1233e2d2016-10-28 13:58:37 +01002840 goto err_unpin;
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002841 }
2842
Chris Wilson0ce81782017-05-17 13:09:59 +01002843 obj->mm.mapping = page_pack_bits(ptr, type);
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002844 }
2845
Chris Wilson1233e2d2016-10-28 13:58:37 +01002846out_unlock:
2847 mutex_unlock(&obj->mm.lock);
Chris Wilsond31d7cb2016-08-12 12:39:58 +01002848 return ptr;
2849
Chris Wilson1233e2d2016-10-28 13:58:37 +01002850err_unpin:
2851 atomic_dec(&obj->mm.pages_pin_count);
2852err_unlock:
2853 ptr = ERR_PTR(ret);
2854 goto out_unlock;
Chris Wilson0a798eb2016-04-08 12:11:11 +01002855}
2856
Chris Wilson7c55e2c2017-03-07 12:03:38 +00002857static int
2858i915_gem_object_pwrite_gtt(struct drm_i915_gem_object *obj,
2859 const struct drm_i915_gem_pwrite *arg)
2860{
2861 struct address_space *mapping = obj->base.filp->f_mapping;
2862 char __user *user_data = u64_to_user_ptr(arg->data_ptr);
2863 u64 remain, offset;
2864 unsigned int pg;
2865
2866 /* Before we instantiate/pin the backing store for our use, we
2867 * can prepopulate the shmemfs filp efficiently using a write into
2868 * the pagecache. We avoid the penalty of instantiating all the
2869 * pages, important if the user is just writing to a few and never
2870 * uses the object on the GPU, and using a direct write into shmemfs
2871 * allows it to avoid the cost of retrieving a page (either swapin
2872 * or clearing-before-use) before it is overwritten.
2873 */
Chris Wilsonf1fa4f42017-10-13 21:26:13 +01002874 if (i915_gem_object_has_pages(obj))
Chris Wilson7c55e2c2017-03-07 12:03:38 +00002875 return -ENODEV;
2876
Chris Wilsona6d65e42017-10-16 21:27:32 +01002877 if (obj->mm.madv != I915_MADV_WILLNEED)
2878 return -EFAULT;
2879
Chris Wilson7c55e2c2017-03-07 12:03:38 +00002880 /* Before the pages are instantiated the object is treated as being
2881 * in the CPU domain. The pages will be clflushed as required before
2882 * use, and we can freely write into the pages directly. If userspace
2883 * races pwrite with any other operation; corruption will ensue -
2884 * that is userspace's prerogative!
2885 */
2886
2887 remain = arg->size;
2888 offset = arg->offset;
2889 pg = offset_in_page(offset);
2890
2891 do {
2892 unsigned int len, unwritten;
2893 struct page *page;
2894 void *data, *vaddr;
2895 int err;
2896
2897 len = PAGE_SIZE - pg;
2898 if (len > remain)
2899 len = remain;
2900
2901 err = pagecache_write_begin(obj->base.filp, mapping,
2902 offset, len, 0,
2903 &page, &data);
2904 if (err < 0)
2905 return err;
2906
2907 vaddr = kmap(page);
2908 unwritten = copy_from_user(vaddr + pg, user_data, len);
2909 kunmap(page);
2910
2911 err = pagecache_write_end(obj->base.filp, mapping,
2912 offset, len, len - unwritten,
2913 page, data);
2914 if (err < 0)
2915 return err;
2916
2917 if (unwritten)
2918 return -EFAULT;
2919
2920 remain -= len;
2921 user_data += len;
2922 offset += len;
2923 pg = 0;
2924 } while (remain);
2925
2926 return 0;
2927}
2928
Mika Kuoppalae5e1fc42016-11-16 17:20:31 +02002929static void i915_gem_context_mark_guilty(struct i915_gem_context *ctx)
Mika Kuoppalaaa60c662013-06-12 15:13:20 +03002930{
Chris Wilson77b25a92017-07-21 13:32:30 +01002931 bool banned;
Mika Kuoppalab083a082016-11-18 15:10:47 +02002932
Chris Wilson77b25a92017-07-21 13:32:30 +01002933 atomic_inc(&ctx->guilty_count);
2934
Chris Wilson24eae082018-02-05 09:22:01 +00002935 banned = false;
2936 if (i915_gem_context_is_bannable(ctx)) {
2937 unsigned int score;
2938
2939 score = atomic_add_return(CONTEXT_SCORE_GUILTY,
2940 &ctx->ban_score);
2941 banned = score >= CONTEXT_SCORE_BAN_THRESHOLD;
2942
2943 DRM_DEBUG_DRIVER("context %s marked guilty (score %d) banned? %s\n",
2944 ctx->name, score, yesno(banned));
2945 }
Chris Wilson77b25a92017-07-21 13:32:30 +01002946 if (!banned)
Mika Kuoppalab083a082016-11-18 15:10:47 +02002947 return;
2948
Chris Wilson77b25a92017-07-21 13:32:30 +01002949 i915_gem_context_set_banned(ctx);
2950 if (!IS_ERR_OR_NULL(ctx->file_priv)) {
2951 atomic_inc(&ctx->file_priv->context_bans);
2952 DRM_DEBUG_DRIVER("client %s has had %d context banned\n",
2953 ctx->name, atomic_read(&ctx->file_priv->context_bans));
2954 }
Mika Kuoppalae5e1fc42016-11-16 17:20:31 +02002955}
2956
2957static void i915_gem_context_mark_innocent(struct i915_gem_context *ctx)
2958{
Chris Wilson77b25a92017-07-21 13:32:30 +01002959 atomic_inc(&ctx->active_count);
Mika Kuoppalaaa60c662013-06-12 15:13:20 +03002960}
2961
Chris Wilsone61e0f52018-02-21 09:56:36 +00002962struct i915_request *
Tvrtko Ursulin0bc40be2016-03-16 11:00:37 +00002963i915_gem_find_active_request(struct intel_engine_cs *engine)
Chris Wilson9375e442010-09-19 12:21:28 +01002964{
Chris Wilsone61e0f52018-02-21 09:56:36 +00002965 struct i915_request *request, *active = NULL;
Chris Wilson754c9fd2017-02-23 07:44:14 +00002966 unsigned long flags;
Mika Kuoppalaaa60c662013-06-12 15:13:20 +03002967
Chris Wilsonf69a02c2016-07-01 17:23:16 +01002968 /* We are called by the error capture and reset at a random
2969 * point in time. In particular, note that neither is crucially
2970 * ordered with an interrupt. After a hang, the GPU is dead and we
2971 * assume that no more writes can happen (we waited long enough for
2972 * all writes that were in transaction to be flushed) - adding an
2973 * extra delay for a recent interrupt is pointless. Hence, we do
2974 * not need an engine->irq_seqno_barrier() before the seqno reads.
2975 */
Chris Wilson754c9fd2017-02-23 07:44:14 +00002976 spin_lock_irqsave(&engine->timeline->lock, flags);
Chris Wilson73cb9702016-10-28 13:58:46 +01002977 list_for_each_entry(request, &engine->timeline->requests, link) {
Chris Wilsone61e0f52018-02-21 09:56:36 +00002978 if (__i915_request_completed(request, request->global_seqno))
Chris Wilson4db080f2013-12-04 11:37:09 +00002979 continue;
Mika Kuoppalaaa60c662013-06-12 15:13:20 +03002980
Mika Kuoppala36193ac2017-01-17 17:59:02 +02002981 GEM_BUG_ON(request->engine != engine);
Chris Wilsonc00122f32017-02-12 17:19:58 +00002982 GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
2983 &request->fence.flags));
Mika Kuoppalab6b0fac2014-01-30 19:04:43 +02002984
Chris Wilson754c9fd2017-02-23 07:44:14 +00002985 active = request;
2986 break;
2987 }
2988 spin_unlock_irqrestore(&engine->timeline->lock, flags);
2989
2990 return active;
Mika Kuoppalab6b0fac2014-01-30 19:04:43 +02002991}
2992
Mika Kuoppalabf2f0432017-01-17 17:59:04 +02002993static bool engine_stalled(struct intel_engine_cs *engine)
2994{
2995 if (!engine->hangcheck.stalled)
2996 return false;
2997
2998 /* Check for possible seqno movement after hang declaration */
2999 if (engine->hangcheck.seqno != intel_engine_get_seqno(engine)) {
3000 DRM_DEBUG_DRIVER("%s pardoned\n", engine->name);
3001 return false;
3002 }
3003
3004 return true;
3005}
3006
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003007/*
3008 * Ensure irq handler finishes, and not run again.
3009 * Also return the active request so that we only search for it once.
3010 */
Chris Wilsone61e0f52018-02-21 09:56:36 +00003011struct i915_request *
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003012i915_gem_reset_prepare_engine(struct intel_engine_cs *engine)
3013{
Chris Wilsone61e0f52018-02-21 09:56:36 +00003014 struct i915_request *request = NULL;
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003015
Chris Wilson1749d902017-10-09 12:02:59 +01003016 /*
3017 * During the reset sequence, we must prevent the engine from
3018 * entering RC6. As the context state is undefined until we restart
3019 * the engine, if it does enter RC6 during the reset, the state
3020 * written to the powercontext is undefined and so we may lose
3021 * GPU state upon resume, i.e. fail to restart after a reset.
3022 */
3023 intel_uncore_forcewake_get(engine->i915, FORCEWAKE_ALL);
3024
3025 /*
3026 * Prevent the signaler thread from updating the request
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003027 * state (by calling dma_fence_signal) as we are processing
3028 * the reset. The write from the GPU of the seqno is
3029 * asynchronous and the signaler thread may see a different
3030 * value to us and declare the request complete, even though
3031 * the reset routine have picked that request as the active
3032 * (incomplete) request. This conflict is not handled
3033 * gracefully!
3034 */
3035 kthread_park(engine->breadcrumbs.signaler);
3036
Chris Wilson1749d902017-10-09 12:02:59 +01003037 /*
3038 * Prevent request submission to the hardware until we have
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003039 * completed the reset in i915_gem_reset_finish(). If a request
3040 * is completed by one engine, it may then queue a request
Sagar Arun Kamblec6dce8f2017-11-16 19:02:37 +05303041 * to a second via its execlists->tasklet *just* as we are
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003042 * calling engine->init_hw() and also writing the ELSP.
Sagar Arun Kamblec6dce8f2017-11-16 19:02:37 +05303043 * Turning off the execlists->tasklet until the reset is over
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003044 * prevents the race.
Chris Wilson68ad3612018-03-07 13:42:26 +00003045 *
3046 * Note that this needs to be a single atomic operation on the
3047 * tasklet (flush existing tasks, prevent new tasks) to prevent
3048 * a race between reset and set-wedged. It is not, so we do the best
3049 * we can atm and make sure we don't lock the machine up in the more
3050 * common case of recursively being called from set-wedged from inside
3051 * i915_reset.
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003052 */
Chris Wilson68ad3612018-03-07 13:42:26 +00003053 if (!atomic_read(&engine->execlists.tasklet.count))
3054 tasklet_kill(&engine->execlists.tasklet);
Sagar Arun Kamblec6dce8f2017-11-16 19:02:37 +05303055 tasklet_disable(&engine->execlists.tasklet);
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003056
Michał Winiarskic41937f2017-10-26 15:35:58 +02003057 /*
3058 * We're using worker to queue preemption requests from the tasklet in
3059 * GuC submission mode.
3060 * Even though tasklet was disabled, we may still have a worker queued.
3061 * Let's make sure that all workers scheduled before disabling the
3062 * tasklet are completed before continuing with the reset.
3063 */
3064 if (engine->i915->guc.preempt_wq)
3065 flush_workqueue(engine->i915->guc.preempt_wq);
3066
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003067 if (engine->irq_seqno_barrier)
3068 engine->irq_seqno_barrier(engine);
3069
Chris Wilsond1d1ebf42017-07-21 13:32:33 +01003070 request = i915_gem_find_active_request(engine);
3071 if (request && request->fence.error == -EIO)
3072 request = ERR_PTR(-EIO); /* Previous reset failed! */
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003073
3074 return request;
3075}
3076
Chris Wilson0e178ae2017-01-17 17:59:06 +02003077int i915_gem_reset_prepare(struct drm_i915_private *dev_priv)
Chris Wilson4c965542017-01-17 17:59:01 +02003078{
3079 struct intel_engine_cs *engine;
Chris Wilsone61e0f52018-02-21 09:56:36 +00003080 struct i915_request *request;
Chris Wilson4c965542017-01-17 17:59:01 +02003081 enum intel_engine_id id;
Chris Wilson0e178ae2017-01-17 17:59:06 +02003082 int err = 0;
Chris Wilson4c965542017-01-17 17:59:01 +02003083
Chris Wilson0e178ae2017-01-17 17:59:06 +02003084 for_each_engine(engine, dev_priv, id) {
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003085 request = i915_gem_reset_prepare_engine(engine);
3086 if (IS_ERR(request)) {
3087 err = PTR_ERR(request);
3088 continue;
Chris Wilson0e178ae2017-01-17 17:59:06 +02003089 }
Michel Thierryc64992e2017-06-20 10:57:44 +01003090
3091 engine->hangcheck.active_request = request;
Chris Wilson0e178ae2017-01-17 17:59:06 +02003092 }
3093
Chris Wilson4c965542017-01-17 17:59:01 +02003094 i915_gem_revoke_fences(dev_priv);
Michal Wajdeczkoc37d5722018-03-12 13:03:07 +00003095 intel_uc_sanitize(dev_priv);
Chris Wilson0e178ae2017-01-17 17:59:06 +02003096
3097 return err;
Chris Wilson4c965542017-01-17 17:59:01 +02003098}
3099
Chris Wilsone61e0f52018-02-21 09:56:36 +00003100static void skip_request(struct i915_request *request)
Mika Kuoppalab6b0fac2014-01-30 19:04:43 +02003101{
Chris Wilson821ed7d2016-09-09 14:11:53 +01003102 void *vaddr = request->ring->vaddr;
3103 u32 head;
Mika Kuoppalab6b0fac2014-01-30 19:04:43 +02003104
Chris Wilson821ed7d2016-09-09 14:11:53 +01003105 /* As this request likely depends on state from the lost
3106 * context, clear out all the user operations leaving the
3107 * breadcrumb at the end (so we get the fence notifications).
3108 */
3109 head = request->head;
3110 if (request->postfix < head) {
3111 memset(vaddr + head, 0, request->ring->size - head);
3112 head = 0;
3113 }
3114 memset(vaddr + head, 0, request->postfix - head);
Chris Wilsonc0d5f322017-01-10 17:22:43 +00003115
3116 dma_fence_set_error(&request->fence, -EIO);
Chris Wilson4db080f2013-12-04 11:37:09 +00003117}
3118
Chris Wilsone61e0f52018-02-21 09:56:36 +00003119static void engine_skip_context(struct i915_request *request)
Mika Kuoppala36193ac2017-01-17 17:59:02 +02003120{
3121 struct intel_engine_cs *engine = request->engine;
3122 struct i915_gem_context *hung_ctx = request->ctx;
3123 struct intel_timeline *timeline;
3124 unsigned long flags;
3125
3126 timeline = i915_gem_context_lookup_timeline(hung_ctx, engine);
3127
3128 spin_lock_irqsave(&engine->timeline->lock, flags);
3129 spin_lock(&timeline->lock);
3130
3131 list_for_each_entry_continue(request, &engine->timeline->requests, link)
3132 if (request->ctx == hung_ctx)
3133 skip_request(request);
3134
3135 list_for_each_entry(request, &timeline->requests, link)
3136 skip_request(request);
3137
3138 spin_unlock(&timeline->lock);
3139 spin_unlock_irqrestore(&engine->timeline->lock, flags);
3140}
3141
Chris Wilsond1d1ebf42017-07-21 13:32:33 +01003142/* Returns the request if it was guilty of the hang */
Chris Wilsone61e0f52018-02-21 09:56:36 +00003143static struct i915_request *
Chris Wilsond1d1ebf42017-07-21 13:32:33 +01003144i915_gem_reset_request(struct intel_engine_cs *engine,
Chris Wilsone61e0f52018-02-21 09:56:36 +00003145 struct i915_request *request)
Mika Kuoppala61da5362017-01-17 17:59:05 +02003146{
Mika Kuoppala71895a02017-01-17 17:59:07 +02003147 /* The guilty request will get skipped on a hung engine.
3148 *
3149 * Users of client default contexts do not rely on logical
3150 * state preserved between batches so it is safe to execute
3151 * queued requests following the hang. Non default contexts
3152 * rely on preserved state, so skipping a batch loses the
3153 * evolution of the state and it needs to be considered corrupted.
3154 * Executing more queued batches on top of corrupted state is
3155 * risky. But we take the risk by trying to advance through
3156 * the queued requests in order to make the client behaviour
3157 * more predictable around resets, by not throwing away random
3158 * amount of batches it has prepared for execution. Sophisticated
3159 * clients can use gem_reset_stats_ioctl and dma fence status
3160 * (exported via sync_file info ioctl on explicit fences) to observe
3161 * when it loses the context state and should rebuild accordingly.
3162 *
3163 * The context ban, and ultimately the client ban, mechanism are safety
3164 * valves if client submission ends up resulting in nothing more than
3165 * subsequent hangs.
3166 */
3167
Chris Wilsond1d1ebf42017-07-21 13:32:33 +01003168 if (engine_stalled(engine)) {
Mika Kuoppala61da5362017-01-17 17:59:05 +02003169 i915_gem_context_mark_guilty(request->ctx);
3170 skip_request(request);
Chris Wilsond1d1ebf42017-07-21 13:32:33 +01003171
3172 /* If this context is now banned, skip all pending requests. */
3173 if (i915_gem_context_is_banned(request->ctx))
3174 engine_skip_context(request);
Mika Kuoppala61da5362017-01-17 17:59:05 +02003175 } else {
Chris Wilsond1d1ebf42017-07-21 13:32:33 +01003176 /*
3177 * Since this is not the hung engine, it may have advanced
3178 * since the hang declaration. Double check by refinding
3179 * the active request at the time of the reset.
3180 */
3181 request = i915_gem_find_active_request(engine);
3182 if (request) {
3183 i915_gem_context_mark_innocent(request->ctx);
3184 dma_fence_set_error(&request->fence, -EAGAIN);
3185
3186 /* Rewind the engine to replay the incomplete rq */
3187 spin_lock_irq(&engine->timeline->lock);
3188 request = list_prev_entry(request, link);
3189 if (&request->link == &engine->timeline->requests)
3190 request = NULL;
3191 spin_unlock_irq(&engine->timeline->lock);
3192 }
Mika Kuoppala61da5362017-01-17 17:59:05 +02003193 }
3194
Chris Wilsond1d1ebf42017-07-21 13:32:33 +01003195 return request;
Mika Kuoppala61da5362017-01-17 17:59:05 +02003196}
3197
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003198void i915_gem_reset_engine(struct intel_engine_cs *engine,
Chris Wilsone61e0f52018-02-21 09:56:36 +00003199 struct i915_request *request)
Chris Wilson4db080f2013-12-04 11:37:09 +00003200{
Chris Wilsonfcb1de52017-12-19 09:01:10 +00003201 /*
3202 * Make sure this write is visible before we re-enable the interrupt
3203 * handlers on another CPU, as tasklet_enable() resolves to just
3204 * a compiler barrier which is insufficient for our purpose here.
3205 */
3206 smp_store_mb(engine->irq_posted, 0);
Chris Wilsoned454f22017-07-21 13:32:29 +01003207
Chris Wilsond1d1ebf42017-07-21 13:32:33 +01003208 if (request)
3209 request = i915_gem_reset_request(engine, request);
3210
3211 if (request) {
Chris Wilsonc0dcb202017-02-07 15:24:37 +00003212 DRM_DEBUG_DRIVER("resetting %s to restart from tail of request 0x%x\n",
3213 engine->name, request->global_seqno);
Chris Wilsonc0dcb202017-02-07 15:24:37 +00003214 }
Chris Wilson821ed7d2016-09-09 14:11:53 +01003215
3216 /* Setup the CS to resume from the breadcrumb of the hung request */
3217 engine->reset_hw(engine, request);
Chris Wilson821ed7d2016-09-09 14:11:53 +01003218}
3219
Chris Wilsond8027092017-02-08 14:30:32 +00003220void i915_gem_reset(struct drm_i915_private *dev_priv)
Chris Wilson821ed7d2016-09-09 14:11:53 +01003221{
3222 struct intel_engine_cs *engine;
Akash Goel3b3f1652016-10-13 22:44:48 +05303223 enum intel_engine_id id;
Chris Wilson821ed7d2016-09-09 14:11:53 +01003224
Chris Wilson4c7d62c2016-10-28 13:58:32 +01003225 lockdep_assert_held(&dev_priv->drm.struct_mutex);
3226
Chris Wilsone61e0f52018-02-21 09:56:36 +00003227 i915_retire_requests(dev_priv);
Chris Wilson821ed7d2016-09-09 14:11:53 +01003228
Chris Wilson2ae55732017-02-12 17:20:02 +00003229 for_each_engine(engine, dev_priv, id) {
3230 struct i915_gem_context *ctx;
3231
Michel Thierryc64992e2017-06-20 10:57:44 +01003232 i915_gem_reset_engine(engine, engine->hangcheck.active_request);
Chris Wilson2ae55732017-02-12 17:20:02 +00003233 ctx = fetch_and_zero(&engine->last_retired_context);
3234 if (ctx)
3235 engine->context_unpin(engine, ctx);
Chris Wilson7b6da812017-12-16 00:03:34 +00003236
3237 /*
3238 * Ostensibily, we always want a context loaded for powersaving,
3239 * so if the engine is idle after the reset, send a request
3240 * to load our scratch kernel_context.
3241 *
3242 * More mysteriously, if we leave the engine idle after a reset,
3243 * the next userspace batch may hang, with what appears to be
3244 * an incoherent read by the CS (presumably stale TLB). An
3245 * empty request appears sufficient to paper over the glitch.
3246 */
Chris Wilson01b8fdc2018-02-05 15:24:31 +00003247 if (intel_engine_is_idle(engine)) {
Chris Wilsone61e0f52018-02-21 09:56:36 +00003248 struct i915_request *rq;
Chris Wilson7b6da812017-12-16 00:03:34 +00003249
Chris Wilsone61e0f52018-02-21 09:56:36 +00003250 rq = i915_request_alloc(engine,
3251 dev_priv->kernel_context);
Chris Wilson7b6da812017-12-16 00:03:34 +00003252 if (!IS_ERR(rq))
Chris Wilsone61e0f52018-02-21 09:56:36 +00003253 __i915_request_add(rq, false);
Chris Wilson7b6da812017-12-16 00:03:34 +00003254 }
Chris Wilson2ae55732017-02-12 17:20:02 +00003255 }
Chris Wilson821ed7d2016-09-09 14:11:53 +01003256
Tvrtko Ursulin4362f4f2016-11-16 08:55:33 +00003257 i915_gem_restore_fences(dev_priv);
Chris Wilsonf2a91d12016-09-21 14:51:06 +01003258
3259 if (dev_priv->gt.awake) {
3260 intel_sanitize_gt_powersave(dev_priv);
3261 intel_enable_gt_powersave(dev_priv);
3262 if (INTEL_GEN(dev_priv) >= 6)
3263 gen6_rps_busy(dev_priv);
3264 }
Chris Wilson821ed7d2016-09-09 14:11:53 +01003265}
3266
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003267void i915_gem_reset_finish_engine(struct intel_engine_cs *engine)
3268{
Sagar Arun Kamblec6dce8f2017-11-16 19:02:37 +05303269 tasklet_enable(&engine->execlists.tasklet);
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003270 kthread_unpark(engine->breadcrumbs.signaler);
Chris Wilson1749d902017-10-09 12:02:59 +01003271
3272 intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL);
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003273}
3274
Chris Wilsond8027092017-02-08 14:30:32 +00003275void i915_gem_reset_finish(struct drm_i915_private *dev_priv)
3276{
Chris Wilson1f7b8472017-02-08 14:30:33 +00003277 struct intel_engine_cs *engine;
3278 enum intel_engine_id id;
3279
Chris Wilsond8027092017-02-08 14:30:32 +00003280 lockdep_assert_held(&dev_priv->drm.struct_mutex);
Chris Wilson1f7b8472017-02-08 14:30:33 +00003281
Chris Wilsonfe3288b2017-02-12 17:20:01 +00003282 for_each_engine(engine, dev_priv, id) {
Michel Thierryc64992e2017-06-20 10:57:44 +01003283 engine->hangcheck.active_request = NULL;
Michel Thierrya1ef70e2017-06-20 10:57:47 +01003284 i915_gem_reset_finish_engine(engine);
Chris Wilsonfe3288b2017-02-12 17:20:01 +00003285 }
Chris Wilsond8027092017-02-08 14:30:32 +00003286}
3287
Chris Wilsone61e0f52018-02-21 09:56:36 +00003288static void nop_submit_request(struct i915_request *request)
Chris Wilson821ed7d2016-09-09 14:11:53 +01003289{
Chris Wilsond9b13c42018-03-15 13:14:50 +00003290 GEM_TRACE("%s fence %llx:%d -> -EIO\n",
3291 request->engine->name,
3292 request->fence.context, request->fence.seqno);
Daniel Vetteraf7a8ff2017-10-11 11:10:19 +02003293 dma_fence_set_error(&request->fence, -EIO);
3294
Chris Wilsone61e0f52018-02-21 09:56:36 +00003295 i915_request_submit(request);
Daniel Vetteraf7a8ff2017-10-11 11:10:19 +02003296}
3297
Chris Wilsone61e0f52018-02-21 09:56:36 +00003298static void nop_complete_submit_request(struct i915_request *request)
Daniel Vetteraf7a8ff2017-10-11 11:10:19 +02003299{
Chris Wilson8d550822017-10-06 12:56:17 +01003300 unsigned long flags;
3301
Chris Wilsond9b13c42018-03-15 13:14:50 +00003302 GEM_TRACE("%s fence %llx:%d -> -EIO\n",
3303 request->engine->name,
3304 request->fence.context, request->fence.seqno);
Chris Wilson3cd94422017-01-10 17:22:45 +00003305 dma_fence_set_error(&request->fence, -EIO);
Chris Wilson8d550822017-10-06 12:56:17 +01003306
3307 spin_lock_irqsave(&request->engine->timeline->lock, flags);
Chris Wilsone61e0f52018-02-21 09:56:36 +00003308 __i915_request_submit(request);
Chris Wilson3dcf93f2016-11-22 14:41:20 +00003309 intel_engine_init_global_seqno(request->engine, request->global_seqno);
Chris Wilson8d550822017-10-06 12:56:17 +01003310 spin_unlock_irqrestore(&request->engine->timeline->lock, flags);
Chris Wilson821ed7d2016-09-09 14:11:53 +01003311}
3312
Daniel Vetteraf7a8ff2017-10-11 11:10:19 +02003313void i915_gem_set_wedged(struct drm_i915_private *i915)
Chris Wilson821ed7d2016-09-09 14:11:53 +01003314{
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00003315 struct intel_engine_cs *engine;
Akash Goel3b3f1652016-10-13 22:44:48 +05303316 enum intel_engine_id id;
Eric Anholt673a3942008-07-30 12:06:12 -07003317
Chris Wilsond9b13c42018-03-15 13:14:50 +00003318 GEM_TRACE("start\n");
3319
Chris Wilson559e0402018-02-05 09:21:59 +00003320 if (drm_debug & DRM_UT_DRIVER) {
3321 struct drm_printer p = drm_debug_printer(__func__);
3322
3323 for_each_engine(engine, i915, id)
3324 intel_engine_dump(engine, &p, "%s\n", engine->name);
3325 }
3326
Chris Wilson0d73e7a2018-02-07 15:13:50 +00003327 set_bit(I915_WEDGED, &i915->gpu_error.flags);
3328 smp_mb__after_atomic();
3329
Daniel Vetteraf7a8ff2017-10-11 11:10:19 +02003330 /*
3331 * First, stop submission to hw, but do not yet complete requests by
3332 * rolling the global seqno forward (since this would complete requests
3333 * for which we haven't set the fence error to EIO yet).
3334 */
Chris Wilson963ddd62018-03-02 11:33:24 +00003335 for_each_engine(engine, i915, id) {
3336 i915_gem_reset_prepare_engine(engine);
Chris Wilson47650db2018-03-07 13:42:25 +00003337
Daniel Vetteraf7a8ff2017-10-11 11:10:19 +02003338 engine->submit_request = nop_submit_request;
Chris Wilson47650db2018-03-07 13:42:25 +00003339 engine->schedule = NULL;
Chris Wilson963ddd62018-03-02 11:33:24 +00003340 }
Chris Wilson47650db2018-03-07 13:42:25 +00003341 i915->caps.scheduler = 0;
Daniel Vetteraf7a8ff2017-10-11 11:10:19 +02003342
Chris Wilsonac697ae2018-03-15 15:10:15 +00003343 /* Even if the GPU reset fails, it should still stop the engines */
3344 intel_gpu_reset(i915, ALL_ENGINES);
3345
Daniel Vetteraf7a8ff2017-10-11 11:10:19 +02003346 /*
3347 * Make sure no one is running the old callback before we proceed with
3348 * cancelling requests and resetting the completion tracking. Otherwise
3349 * we might submit a request to the hardware which never completes.
3350 */
3351 synchronize_rcu();
3352
3353 for_each_engine(engine, i915, id) {
3354 /* Mark all executing requests as skipped */
3355 engine->cancel_requests(engine);
3356
3357 /*
3358 * Only once we've force-cancelled all in-flight requests can we
3359 * start to complete all requests.
3360 */
3361 engine->submit_request = nop_complete_submit_request;
3362 }
3363
3364 /*
3365 * Make sure no request can slip through without getting completed by
3366 * either this call here to intel_engine_init_global_seqno, or the one
3367 * in nop_complete_submit_request.
3368 */
3369 synchronize_rcu();
3370
3371 for_each_engine(engine, i915, id) {
3372 unsigned long flags;
3373
Chris Wilson0d73e7a2018-02-07 15:13:50 +00003374 /*
3375 * Mark all pending requests as complete so that any concurrent
Daniel Vetteraf7a8ff2017-10-11 11:10:19 +02003376 * (lockless) lookup doesn't try and wait upon the request as we
3377 * reset it.
3378 */
3379 spin_lock_irqsave(&engine->timeline->lock, flags);
3380 intel_engine_init_global_seqno(engine,
3381 intel_engine_last_submit(engine));
3382 spin_unlock_irqrestore(&engine->timeline->lock, flags);
Chris Wilson963ddd62018-03-02 11:33:24 +00003383
3384 i915_gem_reset_finish_engine(engine);
Daniel Vetteraf7a8ff2017-10-11 11:10:19 +02003385 }
Chris Wilson20e49332016-11-22 14:41:21 +00003386
Chris Wilsond9b13c42018-03-15 13:14:50 +00003387 GEM_TRACE("end\n");
3388
Chris Wilson3d7adbb2017-07-21 13:32:27 +01003389 wake_up_all(&i915->gpu_error.reset_queue);
Eric Anholt673a3942008-07-30 12:06:12 -07003390}
3391
Chris Wilson2e8f9d32017-03-16 17:13:04 +00003392bool i915_gem_unset_wedged(struct drm_i915_private *i915)
3393{
3394 struct i915_gem_timeline *tl;
3395 int i;
3396
3397 lockdep_assert_held(&i915->drm.struct_mutex);
3398 if (!test_bit(I915_WEDGED, &i915->gpu_error.flags))
3399 return true;
3400
Chris Wilsond9b13c42018-03-15 13:14:50 +00003401 GEM_TRACE("start\n");
3402
Chris Wilson2d4ecac2018-03-07 13:42:21 +00003403 /*
3404 * Before unwedging, make sure that all pending operations
Chris Wilson2e8f9d32017-03-16 17:13:04 +00003405 * are flushed and errored out - we may have requests waiting upon
3406 * third party fences. We marked all inflight requests as EIO, and
3407 * every execbuf since returned EIO, for consistency we want all
3408 * the currently pending requests to also be marked as EIO, which
3409 * is done inside our nop_submit_request - and so we must wait.
3410 *
3411 * No more can be submitted until we reset the wedged bit.
3412 */
3413 list_for_each_entry(tl, &i915->gt.timelines, link) {
3414 for (i = 0; i < ARRAY_SIZE(tl->engine); i++) {
Chris Wilsone61e0f52018-02-21 09:56:36 +00003415 struct i915_request *rq;
Chris Wilson2e8f9d32017-03-16 17:13:04 +00003416
3417 rq = i915_gem_active_peek(&tl->engine[i].last_request,
3418 &i915->drm.struct_mutex);
3419 if (!rq)
3420 continue;
3421
Chris Wilson2d4ecac2018-03-07 13:42:21 +00003422 /*
3423 * We can't use our normal waiter as we want to
Chris Wilson2e8f9d32017-03-16 17:13:04 +00003424 * avoid recursively trying to handle the current
3425 * reset. The basic dma_fence_default_wait() installs
3426 * a callback for dma_fence_signal(), which is
3427 * triggered by our nop handler (indirectly, the
3428 * callback enables the signaler thread which is
3429 * woken by the nop_submit_request() advancing the seqno
3430 * and when the seqno passes the fence, the signaler
3431 * then signals the fence waking us up).
3432 */
3433 if (dma_fence_default_wait(&rq->fence, true,
3434 MAX_SCHEDULE_TIMEOUT) < 0)
3435 return false;
3436 }
3437 }
Chris Wilson2d4ecac2018-03-07 13:42:21 +00003438 i915_retire_requests(i915);
3439 GEM_BUG_ON(i915->gt.active_requests);
Chris Wilson2e8f9d32017-03-16 17:13:04 +00003440
Chris Wilson2d4ecac2018-03-07 13:42:21 +00003441 /*
3442 * Undo nop_submit_request. We prevent all new i915 requests from
Chris Wilson2e8f9d32017-03-16 17:13:04 +00003443 * being queued (by disallowing execbuf whilst wedged) so having
3444 * waited for all active requests above, we know the system is idle
3445 * and do not have to worry about a thread being inside
3446 * engine->submit_request() as we swap over. So unlike installing
3447 * the nop_submit_request on reset, we can do this from normal
3448 * context and do not require stop_machine().
3449 */
3450 intel_engines_reset_default_submission(i915);
Chris Wilson36703e72017-06-22 11:56:25 +01003451 i915_gem_contexts_lost(i915);
Chris Wilson2e8f9d32017-03-16 17:13:04 +00003452
Chris Wilsond9b13c42018-03-15 13:14:50 +00003453 GEM_TRACE("end\n");
3454
Chris Wilson2e8f9d32017-03-16 17:13:04 +00003455 smp_mb__before_atomic(); /* complete takeover before enabling execbuf */
3456 clear_bit(I915_WEDGED, &i915->gpu_error.flags);
3457
3458 return true;
3459}
3460
Daniel Vetter75ef9da2010-08-21 00:25:16 +02003461static void
Eric Anholt673a3942008-07-30 12:06:12 -07003462i915_gem_retire_work_handler(struct work_struct *work)
3463{
Chris Wilsonb29c19b2013-09-25 17:34:56 +01003464 struct drm_i915_private *dev_priv =
Chris Wilson67d97da2016-07-04 08:08:31 +01003465 container_of(work, typeof(*dev_priv), gt.retire_work.work);
Chris Wilson91c8a322016-07-05 10:40:23 +01003466 struct drm_device *dev = &dev_priv->drm;
Eric Anholt673a3942008-07-30 12:06:12 -07003467
Chris Wilson891b48c2010-09-29 12:26:37 +01003468 /* Come back later if the device is busy... */
Chris Wilsonb29c19b2013-09-25 17:34:56 +01003469 if (mutex_trylock(&dev->struct_mutex)) {
Chris Wilsone61e0f52018-02-21 09:56:36 +00003470 i915_retire_requests(dev_priv);
Chris Wilsonb29c19b2013-09-25 17:34:56 +01003471 mutex_unlock(&dev->struct_mutex);
3472 }
Chris Wilson67d97da2016-07-04 08:08:31 +01003473
Chris Wilson88923042018-01-29 14:41:04 +00003474 /*
3475 * Keep the retire handler running until we are finally idle.
Chris Wilson67d97da2016-07-04 08:08:31 +01003476 * We do not need to do this test under locking as in the worst-case
3477 * we queue the retire worker once too often.
3478 */
Chris Wilson88923042018-01-29 14:41:04 +00003479 if (READ_ONCE(dev_priv->gt.awake))
Chris Wilson67d97da2016-07-04 08:08:31 +01003480 queue_delayed_work(dev_priv->wq,
3481 &dev_priv->gt.retire_work,
Chris Wilsonbcb45082012-10-05 17:02:57 +01003482 round_jiffies_up_relative(HZ));
Chris Wilsonb29c19b2013-09-25 17:34:56 +01003483}
Chris Wilson891b48c2010-09-29 12:26:37 +01003484
Chris Wilson84a10742018-01-24 11:36:08 +00003485static void shrink_caches(struct drm_i915_private *i915)
3486{
3487 /*
3488 * kmem_cache_shrink() discards empty slabs and reorders partially
3489 * filled slabs to prioritise allocating from the mostly full slabs,
3490 * with the aim of reducing fragmentation.
3491 */
3492 kmem_cache_shrink(i915->priorities);
3493 kmem_cache_shrink(i915->dependencies);
3494 kmem_cache_shrink(i915->requests);
3495 kmem_cache_shrink(i915->luts);
3496 kmem_cache_shrink(i915->vmas);
3497 kmem_cache_shrink(i915->objects);
3498}
3499
3500struct sleep_rcu_work {
3501 union {
3502 struct rcu_head rcu;
3503 struct work_struct work;
3504 };
3505 struct drm_i915_private *i915;
3506 unsigned int epoch;
3507};
3508
3509static inline bool
3510same_epoch(struct drm_i915_private *i915, unsigned int epoch)
3511{
3512 /*
3513 * There is a small chance that the epoch wrapped since we started
3514 * sleeping. If we assume that epoch is at least a u32, then it will
3515 * take at least 2^32 * 100ms for it to wrap, or about 326 years.
3516 */
3517 return epoch == READ_ONCE(i915->gt.epoch);
3518}
3519
3520static void __sleep_work(struct work_struct *work)
3521{
3522 struct sleep_rcu_work *s = container_of(work, typeof(*s), work);
3523 struct drm_i915_private *i915 = s->i915;
3524 unsigned int epoch = s->epoch;
3525
3526 kfree(s);
3527 if (same_epoch(i915, epoch))
3528 shrink_caches(i915);
3529}
3530
3531static void __sleep_rcu(struct rcu_head *rcu)
3532{
3533 struct sleep_rcu_work *s = container_of(rcu, typeof(*s), rcu);
3534 struct drm_i915_private *i915 = s->i915;
3535
3536 if (same_epoch(i915, s->epoch)) {
3537 INIT_WORK(&s->work, __sleep_work);
3538 queue_work(i915->wq, &s->work);
3539 } else {
3540 kfree(s);
3541 }
3542}
3543
Chris Wilson5427f202017-10-23 22:32:34 +01003544static inline bool
3545new_requests_since_last_retire(const struct drm_i915_private *i915)
3546{
3547 return (READ_ONCE(i915->gt.active_requests) ||
3548 work_pending(&i915->gt.idle_work.work));
3549}
3550
Chris Wilsonb29c19b2013-09-25 17:34:56 +01003551static void
3552i915_gem_idle_work_handler(struct work_struct *work)
3553{
3554 struct drm_i915_private *dev_priv =
Chris Wilson67d97da2016-07-04 08:08:31 +01003555 container_of(work, typeof(*dev_priv), gt.idle_work.work);
Chris Wilson84a10742018-01-24 11:36:08 +00003556 unsigned int epoch = I915_EPOCH_INVALID;
Chris Wilson67d97da2016-07-04 08:08:31 +01003557 bool rearm_hangcheck;
3558
3559 if (!READ_ONCE(dev_priv->gt.awake))
3560 return;
3561
Imre Deak0cb56702016-11-07 11:20:04 +02003562 /*
3563 * Wait for last execlists context complete, but bail out in case a
Chris Wilsonffed7bd2018-03-01 10:33:38 +00003564 * new request is submitted. As we don't trust the hardware, we
3565 * continue on if the wait times out. This is necessary to allow
3566 * the machine to suspend even if the hardware dies, and we will
3567 * try to recover in resume (after depriving the hardware of power,
3568 * it may be in a better mmod).
Imre Deak0cb56702016-11-07 11:20:04 +02003569 */
Chris Wilsonffed7bd2018-03-01 10:33:38 +00003570 __wait_for(if (new_requests_since_last_retire(dev_priv)) return,
3571 intel_engines_are_idle(dev_priv),
3572 I915_IDLE_ENGINES_TIMEOUT * 1000,
3573 10, 500);
Chris Wilson67d97da2016-07-04 08:08:31 +01003574
3575 rearm_hangcheck =
3576 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
3577
Chris Wilson5427f202017-10-23 22:32:34 +01003578 if (!mutex_trylock(&dev_priv->drm.struct_mutex)) {
Chris Wilson67d97da2016-07-04 08:08:31 +01003579 /* Currently busy, come back later */
3580 mod_delayed_work(dev_priv->wq,
3581 &dev_priv->gt.idle_work,
3582 msecs_to_jiffies(50));
3583 goto out_rearm;
3584 }
3585
Imre Deak93c97dc2016-11-07 11:20:03 +02003586 /*
3587 * New request retired after this work handler started, extend active
3588 * period until next instance of the work.
3589 */
Chris Wilson5427f202017-10-23 22:32:34 +01003590 if (new_requests_since_last_retire(dev_priv))
Imre Deak93c97dc2016-11-07 11:20:03 +02003591 goto out_unlock;
3592
Chris Wilsone4d20062018-04-06 16:51:44 +01003593 epoch = __i915_gem_park(dev_priv);
Chris Wilsonff320d62017-10-23 22:32:35 +01003594
Chris Wilson67d97da2016-07-04 08:08:31 +01003595 rearm_hangcheck = false;
Chris Wilson67d97da2016-07-04 08:08:31 +01003596out_unlock:
Chris Wilson5427f202017-10-23 22:32:34 +01003597 mutex_unlock(&dev_priv->drm.struct_mutex);
Chris Wilson35c94182015-04-07 16:20:37 +01003598
Chris Wilson67d97da2016-07-04 08:08:31 +01003599out_rearm:
3600 if (rearm_hangcheck) {
3601 GEM_BUG_ON(!dev_priv->gt.awake);
3602 i915_queue_hangcheck(dev_priv);
Chris Wilson35c94182015-04-07 16:20:37 +01003603 }
Chris Wilson84a10742018-01-24 11:36:08 +00003604
3605 /*
3606 * When we are idle, it is an opportune time to reap our caches.
3607 * However, we have many objects that utilise RCU and the ordered
3608 * i915->wq that this work is executing on. To try and flush any
3609 * pending frees now we are idle, we first wait for an RCU grace
3610 * period, and then queue a task (that will run last on the wq) to
3611 * shrink and re-optimize the caches.
3612 */
3613 if (same_epoch(dev_priv, epoch)) {
3614 struct sleep_rcu_work *s = kmalloc(sizeof(*s), GFP_KERNEL);
3615 if (s) {
3616 s->i915 = dev_priv;
3617 s->epoch = epoch;
3618 call_rcu(&s->rcu, __sleep_rcu);
3619 }
3620 }
Eric Anholt673a3942008-07-30 12:06:12 -07003621}
3622
Chris Wilsonb1f788c2016-08-04 07:52:45 +01003623void i915_gem_close_object(struct drm_gem_object *gem, struct drm_file *file)
3624{
Chris Wilsond1b48c12017-08-16 09:52:08 +01003625 struct drm_i915_private *i915 = to_i915(gem->dev);
Chris Wilsonb1f788c2016-08-04 07:52:45 +01003626 struct drm_i915_gem_object *obj = to_intel_bo(gem);
3627 struct drm_i915_file_private *fpriv = file->driver_priv;
Chris Wilsond1b48c12017-08-16 09:52:08 +01003628 struct i915_lut_handle *lut, *ln;
Chris Wilsonb1f788c2016-08-04 07:52:45 +01003629
Chris Wilsond1b48c12017-08-16 09:52:08 +01003630 mutex_lock(&i915->drm.struct_mutex);
3631
3632 list_for_each_entry_safe(lut, ln, &obj->lut_list, obj_link) {
3633 struct i915_gem_context *ctx = lut->ctx;
3634 struct i915_vma *vma;
3635
Chris Wilson432295d2017-08-22 12:05:15 +01003636 GEM_BUG_ON(ctx->file_priv == ERR_PTR(-EBADF));
Chris Wilsond1b48c12017-08-16 09:52:08 +01003637 if (ctx->file_priv != fpriv)
3638 continue;
3639
3640 vma = radix_tree_delete(&ctx->handles_vma, lut->handle);
Chris Wilson3ffff012017-08-22 12:05:17 +01003641 GEM_BUG_ON(vma->obj != obj);
3642
3643 /* We allow the process to have multiple handles to the same
3644 * vma, in the same fd namespace, by virtue of flink/open.
3645 */
3646 GEM_BUG_ON(!vma->open_count);
3647 if (!--vma->open_count && !i915_vma_is_ggtt(vma))
Chris Wilsonb1f788c2016-08-04 07:52:45 +01003648 i915_vma_close(vma);
Chris Wilsonf8a7fde2016-10-28 13:58:29 +01003649
Chris Wilsond1b48c12017-08-16 09:52:08 +01003650 list_del(&lut->obj_link);
3651 list_del(&lut->ctx_link);
Chris Wilson4ff4b442017-06-16 15:05:16 +01003652
Chris Wilsond1b48c12017-08-16 09:52:08 +01003653 kmem_cache_free(i915->luts, lut);
3654 __i915_gem_object_release_unless_active(obj);
Chris Wilsonf8a7fde2016-10-28 13:58:29 +01003655 }
Chris Wilsond1b48c12017-08-16 09:52:08 +01003656
3657 mutex_unlock(&i915->drm.struct_mutex);
Chris Wilsonb1f788c2016-08-04 07:52:45 +01003658}
3659
Chris Wilsone95433c2016-10-28 13:58:27 +01003660static unsigned long to_wait_timeout(s64 timeout_ns)
3661{
3662 if (timeout_ns < 0)
3663 return MAX_SCHEDULE_TIMEOUT;
3664
3665 if (timeout_ns == 0)
3666 return 0;
3667
3668 return nsecs_to_jiffies_timeout(timeout_ns);
3669}
3670
Ben Widawsky5816d642012-04-11 11:18:19 -07003671/**
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003672 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +01003673 * @dev: drm device pointer
3674 * @data: ioctl data blob
3675 * @file: drm file pointer
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003676 *
3677 * Returns 0 if successful, else an error is returned with the remaining time in
3678 * the timeout parameter.
3679 * -ETIME: object is still busy after timeout
3680 * -ERESTARTSYS: signal interrupted the wait
3681 * -ENONENT: object doesn't exist
3682 * Also possible, but rare:
Chris Wilsonb8050142017-08-11 11:57:31 +01003683 * -EAGAIN: incomplete, restart syscall
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003684 * -ENOMEM: damn
3685 * -ENODEV: Internal IRQ fail
3686 * -E?: The add request failed
3687 *
3688 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
3689 * non-zero timeout parameter the wait ioctl will wait for the given number of
3690 * nanoseconds on an object becoming unbusy. Since the wait itself does so
3691 * without holding struct_mutex the object may become re-busied before this
3692 * function completes. A similar but shorter * race condition exists in the busy
3693 * ioctl
3694 */
3695int
3696i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3697{
3698 struct drm_i915_gem_wait *args = data;
3699 struct drm_i915_gem_object *obj;
Chris Wilsone95433c2016-10-28 13:58:27 +01003700 ktime_t start;
3701 long ret;
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003702
Daniel Vetter11b5d512014-09-29 15:31:26 +02003703 if (args->flags != 0)
3704 return -EINVAL;
3705
Chris Wilson03ac0642016-07-20 13:31:51 +01003706 obj = i915_gem_object_lookup(file, args->bo_handle);
Chris Wilson033d5492016-08-05 10:14:17 +01003707 if (!obj)
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003708 return -ENOENT;
Chris Wilson033d5492016-08-05 10:14:17 +01003709
Chris Wilsone95433c2016-10-28 13:58:27 +01003710 start = ktime_get();
3711
3712 ret = i915_gem_object_wait(obj,
3713 I915_WAIT_INTERRUPTIBLE | I915_WAIT_ALL,
3714 to_wait_timeout(args->timeout_ns),
3715 to_rps_client(file));
3716
3717 if (args->timeout_ns > 0) {
3718 args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
3719 if (args->timeout_ns < 0)
3720 args->timeout_ns = 0;
Chris Wilsonc1d20612017-02-16 12:54:41 +00003721
3722 /*
3723 * Apparently ktime isn't accurate enough and occasionally has a
3724 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
3725 * things up to make the test happy. We allow up to 1 jiffy.
3726 *
3727 * This is a regression from the timespec->ktime conversion.
3728 */
3729 if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
3730 args->timeout_ns = 0;
Chris Wilsonb8050142017-08-11 11:57:31 +01003731
3732 /* Asked to wait beyond the jiffie/scheduler precision? */
3733 if (ret == -ETIME && args->timeout_ns)
3734 ret = -EAGAIN;
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003735 }
3736
Chris Wilsonf0cd5182016-10-28 13:58:43 +01003737 i915_gem_object_put(obj);
John Harrisonff865882014-11-24 18:49:28 +00003738 return ret;
Ben Widawsky23ba4fd2012-05-24 15:03:10 -07003739}
3740
Chris Wilson73cb9702016-10-28 13:58:46 +01003741static int wait_for_timeline(struct i915_gem_timeline *tl, unsigned int flags)
Daniel Vetter4df2faf2010-02-19 11:52:00 +01003742{
Chris Wilson73cb9702016-10-28 13:58:46 +01003743 int ret, i;
3744
3745 for (i = 0; i < ARRAY_SIZE(tl->engine); i++) {
3746 ret = i915_gem_active_wait(&tl->engine[i].last_request, flags);
3747 if (ret)
3748 return ret;
3749 }
3750
3751 return 0;
3752}
3753
Chris Wilson25112b62017-03-30 15:50:39 +01003754static int wait_for_engines(struct drm_i915_private *i915)
3755{
Chris Wilsonee42c002017-12-11 19:41:34 +00003756 if (wait_for(intel_engines_are_idle(i915), I915_IDLE_ENGINES_TIMEOUT)) {
Chris Wilson59e4b192017-12-11 19:41:35 +00003757 dev_err(i915->drm.dev,
3758 "Failed to idle engines, declaring wedged!\n");
Chris Wilson629820f2018-03-09 10:11:14 +00003759 GEM_TRACE_DUMP();
Chris Wilsoncad99462017-08-26 12:09:33 +01003760 i915_gem_set_wedged(i915);
3761 return -EIO;
Chris Wilson25112b62017-03-30 15:50:39 +01003762 }
3763
3764 return 0;
3765}
3766
Chris Wilson73cb9702016-10-28 13:58:46 +01003767int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
3768{
Dave Gordonb4ac5af2016-03-24 11:20:38 +00003769 int ret;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01003770
Chris Wilson863e9fd2017-05-30 13:13:32 +01003771 /* If the device is asleep, we have no requests outstanding */
3772 if (!READ_ONCE(i915->gt.awake))
3773 return 0;
3774
Chris Wilson9caa34a2016-11-11 14:58:08 +00003775 if (flags & I915_WAIT_LOCKED) {
3776 struct i915_gem_timeline *tl;
3777
3778 lockdep_assert_held(&i915->drm.struct_mutex);
3779
3780 list_for_each_entry(tl, &i915->gt.timelines, link) {
3781 ret = wait_for_timeline(tl, flags);
3782 if (ret)
3783 return ret;
3784 }
Chris Wilsone61e0f52018-02-21 09:56:36 +00003785 i915_retire_requests(i915);
Chris Wilson25112b62017-03-30 15:50:39 +01003786
3787 ret = wait_for_engines(i915);
Chris Wilson9caa34a2016-11-11 14:58:08 +00003788 } else {
3789 ret = wait_for_timeline(&i915->gt.global_timeline, flags);
Chris Wilson1ec14ad2010-12-04 11:30:53 +00003790 }
Zou Nan haid1b851f2010-05-21 09:08:57 +08003791
Chris Wilson25112b62017-03-30 15:50:39 +01003792 return ret;
Daniel Vetter4df2faf2010-02-19 11:52:00 +01003793}
3794
Chris Wilson5a97bcc2017-02-22 11:40:46 +00003795static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
3796{
Chris Wilsone27ab732017-06-15 13:38:49 +01003797 /*
3798 * We manually flush the CPU domain so that we can override and
3799 * force the flush for the display, and perform it asyncrhonously.
3800 */
3801 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
3802 if (obj->cache_dirty)
3803 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
Christian Königc0a51fd2018-02-16 13:43:38 +01003804 obj->write_domain = 0;
Chris Wilson5a97bcc2017-02-22 11:40:46 +00003805}
3806
3807void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
3808{
Chris Wilsonbd3d2252017-10-13 21:26:14 +01003809 if (!READ_ONCE(obj->pin_global))
Chris Wilson5a97bcc2017-02-22 11:40:46 +00003810 return;
3811
3812 mutex_lock(&obj->base.dev->struct_mutex);
3813 __i915_gem_object_flush_for_display(obj);
3814 mutex_unlock(&obj->base.dev->struct_mutex);
3815}
3816
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003817/**
Chris Wilsone22d8e32017-04-12 12:01:11 +01003818 * Moves a single object to the WC read, and possibly write domain.
3819 * @obj: object to act on
3820 * @write: ask for write access or read only
3821 *
3822 * This function returns when the move is complete, including waiting on
3823 * flushes to occur.
3824 */
3825int
3826i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
3827{
3828 int ret;
3829
3830 lockdep_assert_held(&obj->base.dev->struct_mutex);
3831
3832 ret = i915_gem_object_wait(obj,
3833 I915_WAIT_INTERRUPTIBLE |
3834 I915_WAIT_LOCKED |
3835 (write ? I915_WAIT_ALL : 0),
3836 MAX_SCHEDULE_TIMEOUT,
3837 NULL);
3838 if (ret)
3839 return ret;
3840
Christian Königc0a51fd2018-02-16 13:43:38 +01003841 if (obj->write_domain == I915_GEM_DOMAIN_WC)
Chris Wilsone22d8e32017-04-12 12:01:11 +01003842 return 0;
3843
3844 /* Flush and acquire obj->pages so that we are coherent through
3845 * direct access in memory with previous cached writes through
3846 * shmemfs and that our cache domain tracking remains valid.
3847 * For example, if the obj->filp was moved to swap without us
3848 * being notified and releasing the pages, we would mistakenly
3849 * continue to assume that the obj remained out of the CPU cached
3850 * domain.
3851 */
3852 ret = i915_gem_object_pin_pages(obj);
3853 if (ret)
3854 return ret;
3855
3856 flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
3857
3858 /* Serialise direct access to this object with the barriers for
3859 * coherent writes from the GPU, by effectively invalidating the
3860 * WC domain upon first access.
3861 */
Christian Königc0a51fd2018-02-16 13:43:38 +01003862 if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)
Chris Wilsone22d8e32017-04-12 12:01:11 +01003863 mb();
3864
3865 /* It should now be out of any other write domains, and we can update
3866 * the domain values for our changes.
3867 */
Christian Königc0a51fd2018-02-16 13:43:38 +01003868 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);
3869 obj->read_domains |= I915_GEM_DOMAIN_WC;
Chris Wilsone22d8e32017-04-12 12:01:11 +01003870 if (write) {
Christian Königc0a51fd2018-02-16 13:43:38 +01003871 obj->read_domains = I915_GEM_DOMAIN_WC;
3872 obj->write_domain = I915_GEM_DOMAIN_WC;
Chris Wilsone22d8e32017-04-12 12:01:11 +01003873 obj->mm.dirty = true;
3874 }
3875
3876 i915_gem_object_unpin_pages(obj);
3877 return 0;
3878}
3879
3880/**
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003881 * Moves a single object to the GTT read, and possibly write domain.
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +01003882 * @obj: object to act on
3883 * @write: ask for write access or read only
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003884 *
3885 * This function returns when the move is complete, including waiting on
3886 * flushes to occur.
3887 */
Jesse Barnes79e53942008-11-07 14:24:08 -08003888int
Chris Wilson20217462010-11-23 15:26:33 +00003889i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003890{
Eric Anholte47c68e2008-11-14 13:35:19 -08003891 int ret;
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003892
Chris Wilsone95433c2016-10-28 13:58:27 +01003893 lockdep_assert_held(&obj->base.dev->struct_mutex);
Chris Wilson4c7d62c2016-10-28 13:58:32 +01003894
Chris Wilsone95433c2016-10-28 13:58:27 +01003895 ret = i915_gem_object_wait(obj,
3896 I915_WAIT_INTERRUPTIBLE |
3897 I915_WAIT_LOCKED |
3898 (write ? I915_WAIT_ALL : 0),
3899 MAX_SCHEDULE_TIMEOUT,
3900 NULL);
Chris Wilson88241782011-01-07 17:09:48 +00003901 if (ret)
3902 return ret;
3903
Christian Königc0a51fd2018-02-16 13:43:38 +01003904 if (obj->write_domain == I915_GEM_DOMAIN_GTT)
Chris Wilsonc13d87e2016-07-20 09:21:15 +01003905 return 0;
3906
Chris Wilson43566de2015-01-02 16:29:29 +05303907 /* Flush and acquire obj->pages so that we are coherent through
3908 * direct access in memory with previous cached writes through
3909 * shmemfs and that our cache domain tracking remains valid.
3910 * For example, if the obj->filp was moved to swap without us
3911 * being notified and releasing the pages, we would mistakenly
3912 * continue to assume that the obj remained out of the CPU cached
3913 * domain.
3914 */
Chris Wilsona4f5ea62016-10-28 13:58:35 +01003915 ret = i915_gem_object_pin_pages(obj);
Chris Wilson43566de2015-01-02 16:29:29 +05303916 if (ret)
3917 return ret;
3918
Chris Wilsonef749212017-04-12 12:01:10 +01003919 flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
Chris Wilson1c5d22f2009-08-25 11:15:50 +01003920
Chris Wilsond0a57782012-10-09 19:24:37 +01003921 /* Serialise direct access to this object with the barriers for
3922 * coherent writes from the GPU, by effectively invalidating the
3923 * GTT domain upon first access.
3924 */
Christian Königc0a51fd2018-02-16 13:43:38 +01003925 if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)
Chris Wilsond0a57782012-10-09 19:24:37 +01003926 mb();
3927
Eric Anholt2ef7eea2008-11-10 10:53:25 -08003928 /* It should now be out of any other write domains, and we can update
3929 * the domain values for our changes.
3930 */
Christian Königc0a51fd2018-02-16 13:43:38 +01003931 GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3932 obj->read_domains |= I915_GEM_DOMAIN_GTT;
Eric Anholte47c68e2008-11-14 13:35:19 -08003933 if (write) {
Christian Königc0a51fd2018-02-16 13:43:38 +01003934 obj->read_domains = I915_GEM_DOMAIN_GTT;
3935 obj->write_domain = I915_GEM_DOMAIN_GTT;
Chris Wilsona4f5ea62016-10-28 13:58:35 +01003936 obj->mm.dirty = true;
Eric Anholte47c68e2008-11-14 13:35:19 -08003937 }
3938
Chris Wilsona4f5ea62016-10-28 13:58:35 +01003939 i915_gem_object_unpin_pages(obj);
Eric Anholte47c68e2008-11-14 13:35:19 -08003940 return 0;
3941}
3942
Chris Wilsonef55f922015-10-09 14:11:27 +01003943/**
3944 * Changes the cache-level of an object across all VMA.
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +01003945 * @obj: object to act on
3946 * @cache_level: new cache level to set for the object
Chris Wilsonef55f922015-10-09 14:11:27 +01003947 *
3948 * After this function returns, the object will be in the new cache-level
3949 * across all GTT and the contents of the backing storage will be coherent,
3950 * with respect to the new cache-level. In order to keep the backing storage
3951 * coherent for all users, we only allow a single cache level to be set
3952 * globally on the object and prevent it from being changed whilst the
3953 * hardware is reading from the object. That is if the object is currently
3954 * on the scanout it will be set to uncached (or equivalent display
3955 * cache coherency) and all non-MOCS GPU access will also be uncached so
3956 * that all direct access to the scanout remains coherent.
3957 */
Chris Wilsone4ffd172011-04-04 09:44:39 +01003958int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3959 enum i915_cache_level cache_level)
3960{
Chris Wilsonaa653a62016-08-04 07:52:27 +01003961 struct i915_vma *vma;
Chris Wilsona6a7cc42016-11-18 21:17:46 +00003962 int ret;
Chris Wilsone4ffd172011-04-04 09:44:39 +01003963
Chris Wilson4c7d62c2016-10-28 13:58:32 +01003964 lockdep_assert_held(&obj->base.dev->struct_mutex);
3965
Chris Wilsone4ffd172011-04-04 09:44:39 +01003966 if (obj->cache_level == cache_level)
Chris Wilsona6a7cc42016-11-18 21:17:46 +00003967 return 0;
Chris Wilsone4ffd172011-04-04 09:44:39 +01003968
Chris Wilsonef55f922015-10-09 14:11:27 +01003969 /* Inspect the list of currently bound VMA and unbind any that would
3970 * be invalid given the new cache-level. This is principally to
3971 * catch the issue of the CS prefetch crossing page boundaries and
3972 * reading an invalid PTE on older architectures.
3973 */
Chris Wilsonaa653a62016-08-04 07:52:27 +01003974restart:
3975 list_for_each_entry(vma, &obj->vma_list, obj_link) {
Chris Wilsonef55f922015-10-09 14:11:27 +01003976 if (!drm_mm_node_allocated(&vma->node))
3977 continue;
3978
Chris Wilson20dfbde2016-08-04 16:32:30 +01003979 if (i915_vma_is_pinned(vma)) {
Chris Wilsonef55f922015-10-09 14:11:27 +01003980 DRM_DEBUG("can not change the cache level of pinned objects\n");
3981 return -EBUSY;
3982 }
3983
Chris Wilson010e3e62017-12-06 12:49:13 +00003984 if (!i915_vma_is_closed(vma) &&
3985 i915_gem_valid_gtt_space(vma, cache_level))
Chris Wilsonaa653a62016-08-04 07:52:27 +01003986 continue;
3987
3988 ret = i915_vma_unbind(vma);
3989 if (ret)
3990 return ret;
3991
3992 /* As unbinding may affect other elements in the
3993 * obj->vma_list (due to side-effects from retiring
3994 * an active vma), play safe and restart the iterator.
3995 */
3996 goto restart;
Chris Wilson42d6ab42012-07-26 11:49:32 +01003997 }
3998
Chris Wilsonef55f922015-10-09 14:11:27 +01003999 /* We can reuse the existing drm_mm nodes but need to change the
4000 * cache-level on the PTE. We could simply unbind them all and
4001 * rebind with the correct cache-level on next use. However since
4002 * we already have a valid slot, dma mapping, pages etc, we may as
4003 * rewrite the PTE in the belief that doing so tramples upon less
4004 * state and so involves less work.
4005 */
Chris Wilson15717de2016-08-04 07:52:26 +01004006 if (obj->bind_count) {
Chris Wilsonef55f922015-10-09 14:11:27 +01004007 /* Before we change the PTE, the GPU must not be accessing it.
4008 * If we wait upon the object, we know that all the bound
4009 * VMA are no longer active.
4010 */
Chris Wilsone95433c2016-10-28 13:58:27 +01004011 ret = i915_gem_object_wait(obj,
4012 I915_WAIT_INTERRUPTIBLE |
4013 I915_WAIT_LOCKED |
4014 I915_WAIT_ALL,
4015 MAX_SCHEDULE_TIMEOUT,
4016 NULL);
Chris Wilsone4ffd172011-04-04 09:44:39 +01004017 if (ret)
4018 return ret;
4019
Tvrtko Ursulin0031fb92016-11-04 14:42:44 +00004020 if (!HAS_LLC(to_i915(obj->base.dev)) &&
4021 cache_level != I915_CACHE_NONE) {
Chris Wilsonef55f922015-10-09 14:11:27 +01004022 /* Access to snoopable pages through the GTT is
4023 * incoherent and on some machines causes a hard
4024 * lockup. Relinquish the CPU mmaping to force
4025 * userspace to refault in the pages and we can
4026 * then double check if the GTT mapping is still
4027 * valid for that pointer access.
4028 */
4029 i915_gem_release_mmap(obj);
Chris Wilsone4ffd172011-04-04 09:44:39 +01004030
Chris Wilsonef55f922015-10-09 14:11:27 +01004031 /* As we no longer need a fence for GTT access,
4032 * we can relinquish it now (and so prevent having
4033 * to steal a fence from someone else on the next
4034 * fence request). Note GPU activity would have
4035 * dropped the fence as all snoopable access is
4036 * supposed to be linear.
4037 */
Chris Wilsone2189dd2017-12-07 21:14:07 +00004038 for_each_ggtt_vma(vma, obj) {
Chris Wilson49ef5292016-08-18 17:17:00 +01004039 ret = i915_vma_put_fence(vma);
4040 if (ret)
4041 return ret;
4042 }
Chris Wilsonef55f922015-10-09 14:11:27 +01004043 } else {
4044 /* We either have incoherent backing store and
4045 * so no GTT access or the architecture is fully
4046 * coherent. In such cases, existing GTT mmaps
4047 * ignore the cache bit in the PTE and we can
4048 * rewrite it without confusing the GPU or having
4049 * to force userspace to fault back in its mmaps.
4050 */
Chris Wilsone4ffd172011-04-04 09:44:39 +01004051 }
4052
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00004053 list_for_each_entry(vma, &obj->vma_list, obj_link) {
Chris Wilsonef55f922015-10-09 14:11:27 +01004054 if (!drm_mm_node_allocated(&vma->node))
4055 continue;
4056
4057 ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
4058 if (ret)
4059 return ret;
4060 }
Chris Wilsone4ffd172011-04-04 09:44:39 +01004061 }
4062
Chris Wilson1c7f4bc2016-02-26 11:03:19 +00004063 list_for_each_entry(vma, &obj->vma_list, obj_link)
Chris Wilson2c225692013-08-09 12:26:45 +01004064 vma->node.color = cache_level;
Chris Wilsonb8f55be2017-08-11 12:11:16 +01004065 i915_gem_object_set_cache_coherency(obj, cache_level);
Chris Wilsone27ab732017-06-15 13:38:49 +01004066 obj->cache_dirty = true; /* Always invalidate stale cachelines */
Chris Wilson2c225692013-08-09 12:26:45 +01004067
Chris Wilsone4ffd172011-04-04 09:44:39 +01004068 return 0;
4069}
4070
Ben Widawsky199adf42012-09-21 17:01:20 -07004071int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
4072 struct drm_file *file)
Chris Wilsone6994ae2012-07-10 10:27:08 +01004073{
Ben Widawsky199adf42012-09-21 17:01:20 -07004074 struct drm_i915_gem_caching *args = data;
Chris Wilsone6994ae2012-07-10 10:27:08 +01004075 struct drm_i915_gem_object *obj;
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004076 int err = 0;
Chris Wilsone6994ae2012-07-10 10:27:08 +01004077
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004078 rcu_read_lock();
4079 obj = i915_gem_object_lookup_rcu(file, args->handle);
4080 if (!obj) {
4081 err = -ENOENT;
4082 goto out;
4083 }
Chris Wilsone6994ae2012-07-10 10:27:08 +01004084
Chris Wilson651d7942013-08-08 14:41:10 +01004085 switch (obj->cache_level) {
4086 case I915_CACHE_LLC:
4087 case I915_CACHE_L3_LLC:
4088 args->caching = I915_CACHING_CACHED;
4089 break;
4090
Chris Wilson4257d3b2013-08-08 14:41:11 +01004091 case I915_CACHE_WT:
4092 args->caching = I915_CACHING_DISPLAY;
4093 break;
4094
Chris Wilson651d7942013-08-08 14:41:10 +01004095 default:
4096 args->caching = I915_CACHING_NONE;
4097 break;
4098 }
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004099out:
4100 rcu_read_unlock();
4101 return err;
Chris Wilsone6994ae2012-07-10 10:27:08 +01004102}
4103
Ben Widawsky199adf42012-09-21 17:01:20 -07004104int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
4105 struct drm_file *file)
Chris Wilsone6994ae2012-07-10 10:27:08 +01004106{
Chris Wilson9c870d02016-10-24 13:42:15 +01004107 struct drm_i915_private *i915 = to_i915(dev);
Ben Widawsky199adf42012-09-21 17:01:20 -07004108 struct drm_i915_gem_caching *args = data;
Chris Wilsone6994ae2012-07-10 10:27:08 +01004109 struct drm_i915_gem_object *obj;
4110 enum i915_cache_level level;
Chris Wilsond65415d2017-01-19 08:22:10 +00004111 int ret = 0;
Chris Wilsone6994ae2012-07-10 10:27:08 +01004112
Ben Widawsky199adf42012-09-21 17:01:20 -07004113 switch (args->caching) {
4114 case I915_CACHING_NONE:
Chris Wilsone6994ae2012-07-10 10:27:08 +01004115 level = I915_CACHE_NONE;
4116 break;
Ben Widawsky199adf42012-09-21 17:01:20 -07004117 case I915_CACHING_CACHED:
Imre Deake5756c12015-08-14 18:43:30 +03004118 /*
4119 * Due to a HW issue on BXT A stepping, GPU stores via a
4120 * snooped mapping may leave stale data in a corresponding CPU
4121 * cacheline, whereas normally such cachelines would get
4122 * invalidated.
4123 */
Chris Wilson9c870d02016-10-24 13:42:15 +01004124 if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
Imre Deake5756c12015-08-14 18:43:30 +03004125 return -ENODEV;
4126
Chris Wilsone6994ae2012-07-10 10:27:08 +01004127 level = I915_CACHE_LLC;
4128 break;
Chris Wilson4257d3b2013-08-08 14:41:11 +01004129 case I915_CACHING_DISPLAY:
Chris Wilson9c870d02016-10-24 13:42:15 +01004130 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
Chris Wilson4257d3b2013-08-08 14:41:11 +01004131 break;
Chris Wilsone6994ae2012-07-10 10:27:08 +01004132 default:
4133 return -EINVAL;
4134 }
4135
Chris Wilsond65415d2017-01-19 08:22:10 +00004136 obj = i915_gem_object_lookup(file, args->handle);
4137 if (!obj)
4138 return -ENOENT;
4139
Tina Zhanga03f3952017-11-14 10:25:13 +00004140 /*
4141 * The caching mode of proxy object is handled by its generator, and
4142 * not allowed to be changed by userspace.
4143 */
4144 if (i915_gem_object_is_proxy(obj)) {
4145 ret = -ENXIO;
4146 goto out;
4147 }
4148
Chris Wilsond65415d2017-01-19 08:22:10 +00004149 if (obj->cache_level == level)
4150 goto out;
4151
4152 ret = i915_gem_object_wait(obj,
4153 I915_WAIT_INTERRUPTIBLE,
4154 MAX_SCHEDULE_TIMEOUT,
4155 to_rps_client(file));
4156 if (ret)
4157 goto out;
4158
Ben Widawsky3bc29132012-09-26 16:15:20 -07004159 ret = i915_mutex_lock_interruptible(dev);
4160 if (ret)
Chris Wilsond65415d2017-01-19 08:22:10 +00004161 goto out;
Chris Wilsone6994ae2012-07-10 10:27:08 +01004162
4163 ret = i915_gem_object_set_cache_level(obj, level);
Chris Wilsone6994ae2012-07-10 10:27:08 +01004164 mutex_unlock(&dev->struct_mutex);
Chris Wilsond65415d2017-01-19 08:22:10 +00004165
4166out:
4167 i915_gem_object_put(obj);
Chris Wilsone6994ae2012-07-10 10:27:08 +01004168 return ret;
4169}
4170
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004171/*
Dhinakaran Pandiyan07bcd992018-03-06 19:34:18 -08004172 * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
4173 * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
4174 * (for pageflips). We only flush the caches while preparing the buffer for
4175 * display, the callers are responsible for frontbuffer flush.
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004176 */
Chris Wilson058d88c2016-08-15 10:49:06 +01004177struct i915_vma *
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004178i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
4179 u32 alignment,
Chris Wilson59354852018-02-20 13:42:06 +00004180 const struct i915_ggtt_view *view,
4181 unsigned int flags)
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004182{
Chris Wilson058d88c2016-08-15 10:49:06 +01004183 struct i915_vma *vma;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004184 int ret;
4185
Chris Wilson4c7d62c2016-10-28 13:58:32 +01004186 lockdep_assert_held(&obj->base.dev->struct_mutex);
4187
Chris Wilsonbd3d2252017-10-13 21:26:14 +01004188 /* Mark the global pin early so that we account for the
Chris Wilsoncc98b412013-08-09 12:25:09 +01004189 * display coherency whilst setting up the cache domains.
4190 */
Chris Wilsonbd3d2252017-10-13 21:26:14 +01004191 obj->pin_global++;
Chris Wilsoncc98b412013-08-09 12:25:09 +01004192
Eric Anholta7ef0642011-03-29 16:59:54 -07004193 /* The display engine is not coherent with the LLC cache on gen6. As
4194 * a result, we make sure that the pinning that is about to occur is
4195 * done with uncached PTEs. This is lowest common denominator for all
4196 * chipsets.
4197 *
4198 * However for gen6+, we could do better by using the GFDT bit instead
4199 * of uncaching, which would allow us to flush all the LLC-cached data
4200 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
4201 */
Chris Wilson651d7942013-08-08 14:41:10 +01004202 ret = i915_gem_object_set_cache_level(obj,
Tvrtko Ursulin86527442016-10-13 11:03:00 +01004203 HAS_WT(to_i915(obj->base.dev)) ?
4204 I915_CACHE_WT : I915_CACHE_NONE);
Chris Wilson058d88c2016-08-15 10:49:06 +01004205 if (ret) {
4206 vma = ERR_PTR(ret);
Chris Wilsonbd3d2252017-10-13 21:26:14 +01004207 goto err_unpin_global;
Chris Wilson058d88c2016-08-15 10:49:06 +01004208 }
Eric Anholta7ef0642011-03-29 16:59:54 -07004209
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004210 /* As the user may map the buffer once pinned in the display plane
4211 * (e.g. libkms for the bootup splash), we have to ensure that we
Chris Wilson2efb8132016-08-18 17:17:06 +01004212 * always use map_and_fenceable for all scanout buffers. However,
4213 * it may simply be too big to fit into mappable, in which case
4214 * put it anyway and hope that userspace can cope (but always first
4215 * try to preserve the existing ABI).
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004216 */
Chris Wilson2efb8132016-08-18 17:17:06 +01004217 vma = ERR_PTR(-ENOSPC);
Chris Wilson59354852018-02-20 13:42:06 +00004218 if ((flags & PIN_MAPPABLE) == 0 &&
4219 (!view || view->type == I915_GGTT_VIEW_NORMAL))
Chris Wilson2efb8132016-08-18 17:17:06 +01004220 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
Chris Wilson59354852018-02-20 13:42:06 +00004221 flags |
4222 PIN_MAPPABLE |
4223 PIN_NONBLOCK);
4224 if (IS_ERR(vma))
Chris Wilson767a2222016-11-07 11:01:28 +00004225 vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags);
Chris Wilson058d88c2016-08-15 10:49:06 +01004226 if (IS_ERR(vma))
Chris Wilsonbd3d2252017-10-13 21:26:14 +01004227 goto err_unpin_global;
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004228
Chris Wilsond8923dc2016-08-18 17:17:07 +01004229 vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
4230
Chris Wilson5a97bcc2017-02-22 11:40:46 +00004231 __i915_gem_object_flush_for_display(obj);
Chris Wilsonb118c1e2010-05-27 13:18:14 +01004232
Chris Wilson2da3b9b2011-04-14 09:41:17 +01004233 /* It should now be out of any other write domains, and we can update
4234 * the domain values for our changes.
4235 */
Christian Königc0a51fd2018-02-16 13:43:38 +01004236 obj->read_domains |= I915_GEM_DOMAIN_GTT;
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004237
Chris Wilson058d88c2016-08-15 10:49:06 +01004238 return vma;
Chris Wilsoncc98b412013-08-09 12:25:09 +01004239
Chris Wilsonbd3d2252017-10-13 21:26:14 +01004240err_unpin_global:
4241 obj->pin_global--;
Chris Wilson058d88c2016-08-15 10:49:06 +01004242 return vma;
Chris Wilsoncc98b412013-08-09 12:25:09 +01004243}
4244
4245void
Chris Wilson058d88c2016-08-15 10:49:06 +01004246i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
Chris Wilsoncc98b412013-08-09 12:25:09 +01004247{
Chris Wilson49d73912016-11-29 09:50:08 +00004248 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
Chris Wilson4c7d62c2016-10-28 13:58:32 +01004249
Chris Wilsonbd3d2252017-10-13 21:26:14 +01004250 if (WARN_ON(vma->obj->pin_global == 0))
Tvrtko Ursulin8a0c39b2015-04-13 11:50:09 +01004251 return;
4252
Chris Wilsonbd3d2252017-10-13 21:26:14 +01004253 if (--vma->obj->pin_global == 0)
Chris Wilsonf51455d2017-01-10 14:47:34 +00004254 vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
Tvrtko Ursuline6617332015-03-23 11:10:33 +00004255
Chris Wilson383d5822016-08-18 17:17:08 +01004256 /* Bump the LRU to try and avoid premature eviction whilst flipping */
Chris Wilsonbefedbb2017-01-19 19:26:55 +00004257 i915_gem_object_bump_inactive_ggtt(vma->obj);
Chris Wilson383d5822016-08-18 17:17:08 +01004258
Chris Wilson058d88c2016-08-15 10:49:06 +01004259 i915_vma_unpin(vma);
Zhenyu Wangb9241ea2009-11-25 13:09:39 +08004260}
4261
Eric Anholte47c68e2008-11-14 13:35:19 -08004262/**
4263 * Moves a single object to the CPU read, and possibly write domain.
Tvrtko Ursulin14bb2c12016-06-03 14:02:17 +01004264 * @obj: object to act on
4265 * @write: requesting write or read-only access
Eric Anholte47c68e2008-11-14 13:35:19 -08004266 *
4267 * This function returns when the move is complete, including waiting on
4268 * flushes to occur.
4269 */
Chris Wilsondabdfe02012-03-26 10:10:27 +02004270int
Chris Wilson919926a2010-11-12 13:42:53 +00004271i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
Eric Anholte47c68e2008-11-14 13:35:19 -08004272{
Eric Anholte47c68e2008-11-14 13:35:19 -08004273 int ret;
4274
Chris Wilsone95433c2016-10-28 13:58:27 +01004275 lockdep_assert_held(&obj->base.dev->struct_mutex);
Chris Wilson4c7d62c2016-10-28 13:58:32 +01004276
Chris Wilsone95433c2016-10-28 13:58:27 +01004277 ret = i915_gem_object_wait(obj,
4278 I915_WAIT_INTERRUPTIBLE |
4279 I915_WAIT_LOCKED |
4280 (write ? I915_WAIT_ALL : 0),
4281 MAX_SCHEDULE_TIMEOUT,
4282 NULL);
Chris Wilson88241782011-01-07 17:09:48 +00004283 if (ret)
4284 return ret;
4285
Chris Wilsonef749212017-04-12 12:01:10 +01004286 flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
Eric Anholte47c68e2008-11-14 13:35:19 -08004287
Eric Anholte47c68e2008-11-14 13:35:19 -08004288 /* Flush the CPU cache if it's still invalid. */
Christian Königc0a51fd2018-02-16 13:43:38 +01004289 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
Chris Wilson57822dc2017-02-22 11:40:48 +00004290 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
Christian Königc0a51fd2018-02-16 13:43:38 +01004291 obj->read_domains |= I915_GEM_DOMAIN_CPU;
Eric Anholte47c68e2008-11-14 13:35:19 -08004292 }
4293
4294 /* It should now be out of any other write domains, and we can update
4295 * the domain values for our changes.
4296 */
Christian Königc0a51fd2018-02-16 13:43:38 +01004297 GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);
Eric Anholte47c68e2008-11-14 13:35:19 -08004298
4299 /* If we're writing through the CPU, then the GPU read domains will
4300 * need to be invalidated at next use.
4301 */
Chris Wilsone27ab732017-06-15 13:38:49 +01004302 if (write)
4303 __start_cpu_write(obj);
Eric Anholt2ef7eea2008-11-10 10:53:25 -08004304
4305 return 0;
4306}
4307
Eric Anholt673a3942008-07-30 12:06:12 -07004308/* Throttle our rendering by waiting until the ring has completed our requests
4309 * emitted over 20 msec ago.
4310 *
Eric Anholtb9624422009-06-03 07:27:35 +00004311 * Note that if we were to use the current jiffies each time around the loop,
4312 * we wouldn't escape the function with any frames outstanding if the time to
4313 * render a frame was over 20ms.
4314 *
Eric Anholt673a3942008-07-30 12:06:12 -07004315 * This should get us reasonable parallelism between CPU and GPU but also
4316 * relatively low latency when blocking on a particular request to finish.
4317 */
4318static int
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004319i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07004320{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004321 struct drm_i915_private *dev_priv = to_i915(dev);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004322 struct drm_i915_file_private *file_priv = file->driver_priv;
Chris Wilsond0bc54f2015-05-21 21:01:48 +01004323 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
Chris Wilsone61e0f52018-02-21 09:56:36 +00004324 struct i915_request *request, *target = NULL;
Chris Wilsone95433c2016-10-28 13:58:27 +01004325 long ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004326
Chris Wilsonf4457ae2016-04-13 17:35:08 +01004327 /* ABI: return -EIO if already wedged */
4328 if (i915_terminally_wedged(&dev_priv->gpu_error))
4329 return -EIO;
Chris Wilsone110e8d2011-01-26 15:39:14 +00004330
Chris Wilson1c255952010-09-26 11:03:27 +01004331 spin_lock(&file_priv->mm.lock);
Chris Wilsonc8659ef2017-03-02 12:25:25 +00004332 list_for_each_entry(request, &file_priv->mm.request_list, client_link) {
Eric Anholtb9624422009-06-03 07:27:35 +00004333 if (time_after_eq(request->emitted_jiffies, recent_enough))
4334 break;
4335
Chris Wilsonc8659ef2017-03-02 12:25:25 +00004336 if (target) {
4337 list_del(&target->client_link);
4338 target->file_priv = NULL;
4339 }
John Harrisonfcfa423c2015-05-29 17:44:12 +01004340
John Harrison54fb2412014-11-24 18:49:27 +00004341 target = request;
Eric Anholtb9624422009-06-03 07:27:35 +00004342 }
John Harrisonff865882014-11-24 18:49:28 +00004343 if (target)
Chris Wilsone61e0f52018-02-21 09:56:36 +00004344 i915_request_get(target);
Chris Wilson1c255952010-09-26 11:03:27 +01004345 spin_unlock(&file_priv->mm.lock);
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004346
John Harrison54fb2412014-11-24 18:49:27 +00004347 if (target == NULL)
Chris Wilsonf787a5f2010-09-24 16:02:42 +01004348 return 0;
4349
Chris Wilsone61e0f52018-02-21 09:56:36 +00004350 ret = i915_request_wait(target,
Chris Wilsone95433c2016-10-28 13:58:27 +01004351 I915_WAIT_INTERRUPTIBLE,
4352 MAX_SCHEDULE_TIMEOUT);
Chris Wilsone61e0f52018-02-21 09:56:36 +00004353 i915_request_put(target);
John Harrisonff865882014-11-24 18:49:28 +00004354
Chris Wilsone95433c2016-10-28 13:58:27 +01004355 return ret < 0 ? ret : 0;
Eric Anholt673a3942008-07-30 12:06:12 -07004356}
4357
Chris Wilson058d88c2016-08-15 10:49:06 +01004358struct i915_vma *
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004359i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
4360 const struct i915_ggtt_view *view,
Chris Wilson91b2db62016-08-04 16:32:23 +01004361 u64 size,
Chris Wilson2ffffd02016-08-04 16:32:22 +01004362 u64 alignment,
4363 u64 flags)
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004364{
Chris Wilsonad16d2e2016-10-13 09:55:04 +01004365 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
4366 struct i915_address_space *vm = &dev_priv->ggtt.base;
Chris Wilson59bfa122016-08-04 16:32:31 +01004367 struct i915_vma *vma;
4368 int ret;
Joonas Lahtinen72e96d62016-03-30 16:57:10 +03004369
Chris Wilson4c7d62c2016-10-28 13:58:32 +01004370 lockdep_assert_held(&obj->base.dev->struct_mutex);
4371
Chris Wilsonac87a6fd2018-02-20 13:42:05 +00004372 if (flags & PIN_MAPPABLE &&
4373 (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
Chris Wilson43ae70d2017-10-09 09:44:01 +01004374 /* If the required space is larger than the available
4375 * aperture, we will not able to find a slot for the
4376 * object and unbinding the object now will be in
4377 * vain. Worse, doing so may cause us to ping-pong
4378 * the object in and out of the Global GTT and
4379 * waste a lot of cycles under the mutex.
4380 */
4381 if (obj->base.size > dev_priv->ggtt.mappable_end)
4382 return ERR_PTR(-E2BIG);
4383
4384 /* If NONBLOCK is set the caller is optimistically
4385 * trying to cache the full object within the mappable
4386 * aperture, and *must* have a fallback in place for
4387 * situations where we cannot bind the object. We
4388 * can be a little more lax here and use the fallback
4389 * more often to avoid costly migrations of ourselves
4390 * and other objects within the aperture.
4391 *
4392 * Half-the-aperture is used as a simple heuristic.
4393 * More interesting would to do search for a free
4394 * block prior to making the commitment to unbind.
4395 * That caters for the self-harm case, and with a
4396 * little more heuristics (e.g. NOFAULT, NOEVICT)
4397 * we could try to minimise harm to others.
4398 */
4399 if (flags & PIN_NONBLOCK &&
4400 obj->base.size > dev_priv->ggtt.mappable_end / 2)
4401 return ERR_PTR(-ENOSPC);
4402 }
4403
Chris Wilson718659a2017-01-16 15:21:28 +00004404 vma = i915_vma_instance(obj, vm, view);
Chris Wilsone0216b72017-01-19 19:26:57 +00004405 if (unlikely(IS_ERR(vma)))
Chris Wilson058d88c2016-08-15 10:49:06 +01004406 return vma;
Chris Wilson59bfa122016-08-04 16:32:31 +01004407
4408 if (i915_vma_misplaced(vma, size, alignment, flags)) {
Chris Wilson43ae70d2017-10-09 09:44:01 +01004409 if (flags & PIN_NONBLOCK) {
4410 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
4411 return ERR_PTR(-ENOSPC);
Chris Wilson59bfa122016-08-04 16:32:31 +01004412
Chris Wilson43ae70d2017-10-09 09:44:01 +01004413 if (flags & PIN_MAPPABLE &&
Chris Wilson944397f2017-01-09 16:16:11 +00004414 vma->fence_size > dev_priv->ggtt.mappable_end / 2)
Chris Wilsonad16d2e2016-10-13 09:55:04 +01004415 return ERR_PTR(-ENOSPC);
4416 }
4417
Chris Wilson59bfa122016-08-04 16:32:31 +01004418 WARN(i915_vma_is_pinned(vma),
4419 "bo is already pinned in ggtt with incorrect alignment:"
Chris Wilson05a20d02016-08-18 17:16:55 +01004420 " offset=%08x, req.alignment=%llx,"
4421 " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n",
4422 i915_ggtt_offset(vma), alignment,
Chris Wilson59bfa122016-08-04 16:32:31 +01004423 !!(flags & PIN_MAPPABLE),
Chris Wilson05a20d02016-08-18 17:16:55 +01004424 i915_vma_is_map_and_fenceable(vma));
Chris Wilson59bfa122016-08-04 16:32:31 +01004425 ret = i915_vma_unbind(vma);
4426 if (ret)
Chris Wilson058d88c2016-08-15 10:49:06 +01004427 return ERR_PTR(ret);
Chris Wilson59bfa122016-08-04 16:32:31 +01004428 }
4429
Chris Wilson058d88c2016-08-15 10:49:06 +01004430 ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
4431 if (ret)
4432 return ERR_PTR(ret);
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02004433
Chris Wilson058d88c2016-08-15 10:49:06 +01004434 return vma;
Eric Anholt673a3942008-07-30 12:06:12 -07004435}
4436
Chris Wilsonedf6b762016-08-09 09:23:33 +01004437static __always_inline unsigned int __busy_read_flag(unsigned int id)
Chris Wilson3fdc13c2016-08-05 10:14:18 +01004438{
4439 /* Note that we could alias engines in the execbuf API, but
4440 * that would be very unwise as it prevents userspace from
4441 * fine control over engine selection. Ahem.
4442 *
4443 * This should be something like EXEC_MAX_ENGINE instead of
4444 * I915_NUM_ENGINES.
4445 */
4446 BUILD_BUG_ON(I915_NUM_ENGINES > 16);
4447 return 0x10000 << id;
4448}
4449
4450static __always_inline unsigned int __busy_write_id(unsigned int id)
4451{
Chris Wilson70cb4722016-08-09 18:08:25 +01004452 /* The uABI guarantees an active writer is also amongst the read
4453 * engines. This would be true if we accessed the activity tracking
4454 * under the lock, but as we perform the lookup of the object and
4455 * its activity locklessly we can not guarantee that the last_write
4456 * being active implies that we have set the same engine flag from
4457 * last_read - hence we always set both read and write busy for
4458 * last_write.
4459 */
4460 return id | __busy_read_flag(id);
Chris Wilson3fdc13c2016-08-05 10:14:18 +01004461}
4462
Chris Wilsonedf6b762016-08-09 09:23:33 +01004463static __always_inline unsigned int
Chris Wilsond07f0e52016-10-28 13:58:44 +01004464__busy_set_if_active(const struct dma_fence *fence,
Chris Wilson3fdc13c2016-08-05 10:14:18 +01004465 unsigned int (*flag)(unsigned int id))
4466{
Chris Wilsone61e0f52018-02-21 09:56:36 +00004467 struct i915_request *rq;
Chris Wilson12555012016-08-16 09:50:40 +01004468
Chris Wilsond07f0e52016-10-28 13:58:44 +01004469 /* We have to check the current hw status of the fence as the uABI
4470 * guarantees forward progress. We could rely on the idle worker
4471 * to eventually flush us, but to minimise latency just ask the
4472 * hardware.
4473 *
4474 * Note we only report on the status of native fences.
4475 */
4476 if (!dma_fence_is_i915(fence))
Chris Wilson12555012016-08-16 09:50:40 +01004477 return 0;
4478
Chris Wilsond07f0e52016-10-28 13:58:44 +01004479 /* opencode to_request() in order to avoid const warnings */
Chris Wilsone61e0f52018-02-21 09:56:36 +00004480 rq = container_of(fence, struct i915_request, fence);
4481 if (i915_request_completed(rq))
Chris Wilsond07f0e52016-10-28 13:58:44 +01004482 return 0;
4483
Chris Wilson1d39f282017-04-11 13:43:06 +01004484 return flag(rq->engine->uabi_id);
Chris Wilson3fdc13c2016-08-05 10:14:18 +01004485}
4486
Chris Wilsonedf6b762016-08-09 09:23:33 +01004487static __always_inline unsigned int
Chris Wilsond07f0e52016-10-28 13:58:44 +01004488busy_check_reader(const struct dma_fence *fence)
Chris Wilson3fdc13c2016-08-05 10:14:18 +01004489{
Chris Wilsond07f0e52016-10-28 13:58:44 +01004490 return __busy_set_if_active(fence, __busy_read_flag);
Chris Wilson3fdc13c2016-08-05 10:14:18 +01004491}
4492
Chris Wilsonedf6b762016-08-09 09:23:33 +01004493static __always_inline unsigned int
Chris Wilsond07f0e52016-10-28 13:58:44 +01004494busy_check_writer(const struct dma_fence *fence)
Chris Wilson3fdc13c2016-08-05 10:14:18 +01004495{
Chris Wilsond07f0e52016-10-28 13:58:44 +01004496 if (!fence)
4497 return 0;
4498
4499 return __busy_set_if_active(fence, __busy_write_id);
Chris Wilson3fdc13c2016-08-05 10:14:18 +01004500}
4501
Eric Anholt673a3942008-07-30 12:06:12 -07004502int
Eric Anholt673a3942008-07-30 12:06:12 -07004503i915_gem_busy_ioctl(struct drm_device *dev, void *data,
Chris Wilson05394f32010-11-08 19:18:58 +00004504 struct drm_file *file)
Eric Anholt673a3942008-07-30 12:06:12 -07004505{
4506 struct drm_i915_gem_busy *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00004507 struct drm_i915_gem_object *obj;
Chris Wilsond07f0e52016-10-28 13:58:44 +01004508 struct reservation_object_list *list;
4509 unsigned int seq;
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004510 int err;
Eric Anholt673a3942008-07-30 12:06:12 -07004511
Chris Wilsond07f0e52016-10-28 13:58:44 +01004512 err = -ENOENT;
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004513 rcu_read_lock();
4514 obj = i915_gem_object_lookup_rcu(file, args->handle);
Chris Wilsond07f0e52016-10-28 13:58:44 +01004515 if (!obj)
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004516 goto out;
Chris Wilsond07f0e52016-10-28 13:58:44 +01004517
4518 /* A discrepancy here is that we do not report the status of
4519 * non-i915 fences, i.e. even though we may report the object as idle,
4520 * a call to set-domain may still stall waiting for foreign rendering.
4521 * This also means that wait-ioctl may report an object as busy,
4522 * where busy-ioctl considers it idle.
4523 *
4524 * We trade the ability to warn of foreign fences to report on which
4525 * i915 engines are active for the object.
4526 *
4527 * Alternatively, we can trade that extra information on read/write
4528 * activity with
4529 * args->busy =
4530 * !reservation_object_test_signaled_rcu(obj->resv, true);
4531 * to report the overall busyness. This is what the wait-ioctl does.
4532 *
4533 */
4534retry:
4535 seq = raw_read_seqcount(&obj->resv->seq);
4536
4537 /* Translate the exclusive fence to the READ *and* WRITE engine */
4538 args->busy = busy_check_writer(rcu_dereference(obj->resv->fence_excl));
4539
4540 /* Translate shared fences to READ set of engines */
4541 list = rcu_dereference(obj->resv->fence);
4542 if (list) {
4543 unsigned int shared_count = list->shared_count, i;
4544
4545 for (i = 0; i < shared_count; ++i) {
4546 struct dma_fence *fence =
4547 rcu_dereference(list->shared[i]);
4548
4549 args->busy |= busy_check_reader(fence);
4550 }
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004551 }
Zou Nan haid1b851f2010-05-21 09:08:57 +08004552
Chris Wilsond07f0e52016-10-28 13:58:44 +01004553 if (args->busy && read_seqcount_retry(&obj->resv->seq, seq))
4554 goto retry;
Chris Wilson426960b2016-01-15 16:51:46 +00004555
Chris Wilsond07f0e52016-10-28 13:58:44 +01004556 err = 0;
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004557out:
4558 rcu_read_unlock();
4559 return err;
Eric Anholt673a3942008-07-30 12:06:12 -07004560}
4561
4562int
4563i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4564 struct drm_file *file_priv)
4565{
Akshay Joshi0206e352011-08-16 15:34:10 -04004566 return i915_gem_ring_throttle(dev, file_priv);
Eric Anholt673a3942008-07-30 12:06:12 -07004567}
4568
Chris Wilson3ef94da2009-09-14 16:50:29 +01004569int
4570i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4571 struct drm_file *file_priv)
4572{
Chris Wilsonfac5e232016-07-04 11:34:36 +01004573 struct drm_i915_private *dev_priv = to_i915(dev);
Chris Wilson3ef94da2009-09-14 16:50:29 +01004574 struct drm_i915_gem_madvise *args = data;
Chris Wilson05394f32010-11-08 19:18:58 +00004575 struct drm_i915_gem_object *obj;
Chris Wilson1233e2d2016-10-28 13:58:37 +01004576 int err;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004577
4578 switch (args->madv) {
4579 case I915_MADV_DONTNEED:
4580 case I915_MADV_WILLNEED:
4581 break;
4582 default:
4583 return -EINVAL;
4584 }
4585
Chris Wilson03ac0642016-07-20 13:31:51 +01004586 obj = i915_gem_object_lookup(file_priv, args->handle);
Chris Wilson1233e2d2016-10-28 13:58:37 +01004587 if (!obj)
4588 return -ENOENT;
4589
4590 err = mutex_lock_interruptible(&obj->mm.lock);
4591 if (err)
4592 goto out;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004593
Chris Wilsonf1fa4f42017-10-13 21:26:13 +01004594 if (i915_gem_object_has_pages(obj) &&
Chris Wilson3e510a82016-08-05 10:14:23 +01004595 i915_gem_object_is_tiled(obj) &&
Daniel Vetter656bfa32014-11-20 09:26:30 +01004596 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
Chris Wilsonbc0629a2016-11-01 10:03:17 +00004597 if (obj->mm.madv == I915_MADV_WILLNEED) {
4598 GEM_BUG_ON(!obj->mm.quirked);
Chris Wilsona4f5ea62016-10-28 13:58:35 +01004599 __i915_gem_object_unpin_pages(obj);
Chris Wilsonbc0629a2016-11-01 10:03:17 +00004600 obj->mm.quirked = false;
4601 }
4602 if (args->madv == I915_MADV_WILLNEED) {
Chris Wilson2c3a3f42016-11-04 10:30:01 +00004603 GEM_BUG_ON(obj->mm.quirked);
Chris Wilsona4f5ea62016-10-28 13:58:35 +01004604 __i915_gem_object_pin_pages(obj);
Chris Wilsonbc0629a2016-11-01 10:03:17 +00004605 obj->mm.quirked = true;
4606 }
Daniel Vetter656bfa32014-11-20 09:26:30 +01004607 }
4608
Chris Wilsona4f5ea62016-10-28 13:58:35 +01004609 if (obj->mm.madv != __I915_MADV_PURGED)
4610 obj->mm.madv = args->madv;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004611
Chris Wilson6c085a72012-08-20 11:40:46 +02004612 /* if the object is no longer attached, discard its backing storage */
Chris Wilsonf1fa4f42017-10-13 21:26:13 +01004613 if (obj->mm.madv == I915_MADV_DONTNEED &&
4614 !i915_gem_object_has_pages(obj))
Chris Wilson2d7ef392009-09-20 23:13:10 +01004615 i915_gem_object_truncate(obj);
4616
Chris Wilsona4f5ea62016-10-28 13:58:35 +01004617 args->retained = obj->mm.madv != __I915_MADV_PURGED;
Chris Wilson1233e2d2016-10-28 13:58:37 +01004618 mutex_unlock(&obj->mm.lock);
Chris Wilsonbb6baf72009-09-22 14:24:13 +01004619
Chris Wilson1233e2d2016-10-28 13:58:37 +01004620out:
Chris Wilsonf8c417c2016-07-20 13:31:53 +01004621 i915_gem_object_put(obj);
Chris Wilson1233e2d2016-10-28 13:58:37 +01004622 return err;
Chris Wilson3ef94da2009-09-14 16:50:29 +01004623}
4624
Chris Wilson5b8c8ae2016-11-16 19:07:04 +00004625static void
Chris Wilsone61e0f52018-02-21 09:56:36 +00004626frontbuffer_retire(struct i915_gem_active *active, struct i915_request *request)
Chris Wilson5b8c8ae2016-11-16 19:07:04 +00004627{
4628 struct drm_i915_gem_object *obj =
4629 container_of(active, typeof(*obj), frontbuffer_write);
4630
Chris Wilsond59b21e2017-02-22 11:40:49 +00004631 intel_fb_obj_flush(obj, ORIGIN_CS);
Chris Wilson5b8c8ae2016-11-16 19:07:04 +00004632}
4633
Chris Wilson37e680a2012-06-07 15:38:42 +01004634void i915_gem_object_init(struct drm_i915_gem_object *obj,
4635 const struct drm_i915_gem_object_ops *ops)
Chris Wilson0327d6b2012-08-11 15:41:06 +01004636{
Chris Wilson1233e2d2016-10-28 13:58:37 +01004637 mutex_init(&obj->mm.lock);
4638
Ben Widawsky2f633152013-07-17 12:19:03 -07004639 INIT_LIST_HEAD(&obj->vma_list);
Chris Wilsond1b48c12017-08-16 09:52:08 +01004640 INIT_LIST_HEAD(&obj->lut_list);
Chris Wilson8d9d5742015-04-07 16:20:38 +01004641 INIT_LIST_HEAD(&obj->batch_pool_link);
Chris Wilson0327d6b2012-08-11 15:41:06 +01004642
Chris Wilson37e680a2012-06-07 15:38:42 +01004643 obj->ops = ops;
4644
Chris Wilsond07f0e52016-10-28 13:58:44 +01004645 reservation_object_init(&obj->__builtin_resv);
4646 obj->resv = &obj->__builtin_resv;
4647
Chris Wilson50349242016-08-18 17:17:04 +01004648 obj->frontbuffer_ggtt_origin = ORIGIN_GTT;
Chris Wilson5b8c8ae2016-11-16 19:07:04 +00004649 init_request_active(&obj->frontbuffer_write, frontbuffer_retire);
Chris Wilsona4f5ea62016-10-28 13:58:35 +01004650
4651 obj->mm.madv = I915_MADV_WILLNEED;
4652 INIT_RADIX_TREE(&obj->mm.get_page.radix, GFP_KERNEL | __GFP_NOWARN);
4653 mutex_init(&obj->mm.get_page.lock);
Chris Wilson0327d6b2012-08-11 15:41:06 +01004654
Dave Gordonf19ec8c2016-07-04 11:34:37 +01004655 i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
Chris Wilson0327d6b2012-08-11 15:41:06 +01004656}
4657
Chris Wilson37e680a2012-06-07 15:38:42 +01004658static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
Tvrtko Ursulin3599a912016-11-01 14:44:10 +00004659 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE |
4660 I915_GEM_OBJECT_IS_SHRINKABLE,
Chris Wilson7c55e2c2017-03-07 12:03:38 +00004661
Chris Wilson37e680a2012-06-07 15:38:42 +01004662 .get_pages = i915_gem_object_get_pages_gtt,
4663 .put_pages = i915_gem_object_put_pages_gtt,
Chris Wilson7c55e2c2017-03-07 12:03:38 +00004664
4665 .pwrite = i915_gem_object_pwrite_gtt,
Chris Wilson37e680a2012-06-07 15:38:42 +01004666};
4667
Matthew Auld465c4032017-10-06 23:18:14 +01004668static int i915_gem_object_create_shmem(struct drm_device *dev,
4669 struct drm_gem_object *obj,
4670 size_t size)
4671{
4672 struct drm_i915_private *i915 = to_i915(dev);
4673 unsigned long flags = VM_NORESERVE;
4674 struct file *filp;
4675
4676 drm_gem_private_object_init(dev, obj, size);
4677
4678 if (i915->mm.gemfs)
4679 filp = shmem_file_setup_with_mnt(i915->mm.gemfs, "i915", size,
4680 flags);
4681 else
4682 filp = shmem_file_setup("i915", size, flags);
4683
4684 if (IS_ERR(filp))
4685 return PTR_ERR(filp);
4686
4687 obj->filp = filp;
4688
4689 return 0;
4690}
4691
Chris Wilsonb4bcbe22016-10-18 13:02:49 +01004692struct drm_i915_gem_object *
Tvrtko Ursulin12d79d72016-12-01 14:16:37 +00004693i915_gem_object_create(struct drm_i915_private *dev_priv, u64 size)
Daniel Vetterac52bc52010-04-09 19:05:06 +00004694{
Daniel Vetterc397b902010-04-09 19:05:07 +00004695 struct drm_i915_gem_object *obj;
Hugh Dickins5949eac2011-06-27 16:18:18 -07004696 struct address_space *mapping;
Chris Wilsonb8f55be2017-08-11 12:11:16 +01004697 unsigned int cache_level;
Daniel Vetter1a240d42012-11-29 22:18:51 +01004698 gfp_t mask;
Chris Wilsonfe3db792016-04-25 13:32:13 +01004699 int ret;
Daniel Vetterc397b902010-04-09 19:05:07 +00004700
Chris Wilsonb4bcbe22016-10-18 13:02:49 +01004701 /* There is a prevalence of the assumption that we fit the object's
4702 * page count inside a 32bit _signed_ variable. Let's document this and
4703 * catch if we ever need to fix it. In the meantime, if you do spot
4704 * such a local variable, please consider fixing!
4705 */
Tvrtko Ursulin7a3ee5d2017-03-30 17:31:30 +01004706 if (size >> PAGE_SHIFT > INT_MAX)
Chris Wilsonb4bcbe22016-10-18 13:02:49 +01004707 return ERR_PTR(-E2BIG);
4708
4709 if (overflows_type(size, obj->base.size))
4710 return ERR_PTR(-E2BIG);
4711
Tvrtko Ursulin187685c2016-12-01 14:16:36 +00004712 obj = i915_gem_object_alloc(dev_priv);
Daniel Vetterc397b902010-04-09 19:05:07 +00004713 if (obj == NULL)
Chris Wilsonfe3db792016-04-25 13:32:13 +01004714 return ERR_PTR(-ENOMEM);
Daniel Vetterc397b902010-04-09 19:05:07 +00004715
Matthew Auld465c4032017-10-06 23:18:14 +01004716 ret = i915_gem_object_create_shmem(&dev_priv->drm, &obj->base, size);
Chris Wilsonfe3db792016-04-25 13:32:13 +01004717 if (ret)
4718 goto fail;
Daniel Vetterc397b902010-04-09 19:05:07 +00004719
Chris Wilsonbed1ea92012-05-24 20:48:12 +01004720 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
Jani Nikulac0f86832016-12-07 12:13:04 +02004721 if (IS_I965GM(dev_priv) || IS_I965G(dev_priv)) {
Chris Wilsonbed1ea92012-05-24 20:48:12 +01004722 /* 965gm cannot relocate objects above 4GiB. */
4723 mask &= ~__GFP_HIGHMEM;
4724 mask |= __GFP_DMA32;
4725 }
4726
Al Viro93c76a32015-12-04 23:45:44 -05004727 mapping = obj->base.filp->f_mapping;
Chris Wilsonbed1ea92012-05-24 20:48:12 +01004728 mapping_set_gfp_mask(mapping, mask);
Chris Wilson4846bf02017-06-09 12:03:46 +01004729 GEM_BUG_ON(!(mapping_gfp_mask(mapping) & __GFP_RECLAIM));
Hugh Dickins5949eac2011-06-27 16:18:18 -07004730
Chris Wilson37e680a2012-06-07 15:38:42 +01004731 i915_gem_object_init(obj, &i915_gem_object_ops);
Chris Wilson73aa8082010-09-30 11:46:12 +01004732
Christian Königc0a51fd2018-02-16 13:43:38 +01004733 obj->write_domain = I915_GEM_DOMAIN_CPU;
4734 obj->read_domains = I915_GEM_DOMAIN_CPU;
Daniel Vetterc397b902010-04-09 19:05:07 +00004735
Chris Wilsonb8f55be2017-08-11 12:11:16 +01004736 if (HAS_LLC(dev_priv))
Eugeni Dodonov3d29b842012-01-17 14:43:53 -02004737 /* On some devices, we can have the GPU use the LLC (the CPU
Eric Anholta1871112011-03-29 16:59:55 -07004738 * cache) for about a 10% performance improvement
4739 * compared to uncached. Graphics requests other than
4740 * display scanout are coherent with the CPU in
4741 * accessing this cache. This means in this mode we
4742 * don't need to clflush on the CPU side, and on the
4743 * GPU side we only need to flush internal caches to
4744 * get data visible to the CPU.
4745 *
4746 * However, we maintain the display planes as UC, and so
4747 * need to rebind when first used as such.
4748 */
Chris Wilsonb8f55be2017-08-11 12:11:16 +01004749 cache_level = I915_CACHE_LLC;
4750 else
4751 cache_level = I915_CACHE_NONE;
Eric Anholta1871112011-03-29 16:59:55 -07004752
Chris Wilsonb8f55be2017-08-11 12:11:16 +01004753 i915_gem_object_set_cache_coherency(obj, cache_level);
Chris Wilsone27ab732017-06-15 13:38:49 +01004754
Daniel Vetterd861e332013-07-24 23:25:03 +02004755 trace_i915_gem_object_create(obj);
4756
Chris Wilson05394f32010-11-08 19:18:58 +00004757 return obj;
Chris Wilsonfe3db792016-04-25 13:32:13 +01004758
4759fail:
4760 i915_gem_object_free(obj);
Chris Wilsonfe3db792016-04-25 13:32:13 +01004761 return ERR_PTR(ret);
Daniel Vetterac52bc52010-04-09 19:05:06 +00004762}
4763
Chris Wilson340fbd82014-05-22 09:16:52 +01004764static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4765{
4766 /* If we are the last user of the backing storage (be it shmemfs
4767 * pages or stolen etc), we know that the pages are going to be
4768 * immediately released. In this case, we can then skip copying
4769 * back the contents from the GPU.
4770 */
4771
Chris Wilsona4f5ea62016-10-28 13:58:35 +01004772 if (obj->mm.madv != I915_MADV_WILLNEED)
Chris Wilson340fbd82014-05-22 09:16:52 +01004773 return false;
4774
4775 if (obj->base.filp == NULL)
4776 return true;
4777
4778 /* At first glance, this looks racy, but then again so would be
4779 * userspace racing mmap against close. However, the first external
4780 * reference to the filp can only be obtained through the
4781 * i915_gem_mmap_ioctl() which safeguards us against the user
4782 * acquiring such a reference whilst we are in the middle of
4783 * freeing the object.
4784 */
4785 return atomic_long_read(&obj->base.filp->f_count) == 1;
4786}
4787
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004788static void __i915_gem_free_objects(struct drm_i915_private *i915,
4789 struct llist_node *freed)
Chris Wilsonbe726152010-07-23 23:18:50 +01004790{
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004791 struct drm_i915_gem_object *obj, *on;
Chris Wilsonbe726152010-07-23 23:18:50 +01004792
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004793 intel_runtime_pm_get(i915);
Chris Wilsoncc731f52017-10-13 21:26:21 +01004794 llist_for_each_entry_safe(obj, on, freed, freed) {
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004795 struct i915_vma *vma, *vn;
Paulo Zanonif65c9162013-11-27 18:20:34 -02004796
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004797 trace_i915_gem_object_destroy(obj);
4798
Chris Wilsoncc731f52017-10-13 21:26:21 +01004799 mutex_lock(&i915->drm.struct_mutex);
4800
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004801 GEM_BUG_ON(i915_gem_object_is_active(obj));
4802 list_for_each_entry_safe(vma, vn,
4803 &obj->vma_list, obj_link) {
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004804 GEM_BUG_ON(i915_vma_is_active(vma));
4805 vma->flags &= ~I915_VMA_PIN_MASK;
4806 i915_vma_close(vma);
4807 }
Chris Wilsondb6c2b42016-11-01 11:54:00 +00004808 GEM_BUG_ON(!list_empty(&obj->vma_list));
4809 GEM_BUG_ON(!RB_EMPTY_ROOT(&obj->vma_tree));
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004810
Chris Wilsonf2123812017-10-16 12:40:37 +01004811 /* This serializes freeing with the shrinker. Since the free
4812 * is delayed, first by RCU then by the workqueue, we want the
4813 * shrinker to be able to free pages of unreferenced objects,
4814 * or else we may oom whilst there are plenty of deferred
4815 * freed objects.
4816 */
4817 if (i915_gem_object_has_pages(obj)) {
4818 spin_lock(&i915->mm.obj_lock);
4819 list_del_init(&obj->mm.link);
4820 spin_unlock(&i915->mm.obj_lock);
4821 }
4822
Chris Wilsoncc731f52017-10-13 21:26:21 +01004823 mutex_unlock(&i915->drm.struct_mutex);
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004824
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004825 GEM_BUG_ON(obj->bind_count);
Chris Wilsona65adaf2017-10-09 09:43:57 +01004826 GEM_BUG_ON(obj->userfault_count);
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004827 GEM_BUG_ON(atomic_read(&obj->frontbuffer_bits));
Chris Wilson67b48042017-08-22 12:05:16 +01004828 GEM_BUG_ON(!list_empty(&obj->lut_list));
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004829
4830 if (obj->ops->release)
4831 obj->ops->release(obj);
4832
4833 if (WARN_ON(i915_gem_object_has_pinned_pages(obj)))
4834 atomic_set(&obj->mm.pages_pin_count, 0);
Chris Wilson548625e2016-11-01 12:11:34 +00004835 __i915_gem_object_put_pages(obj, I915_MM_NORMAL);
Chris Wilsonf1fa4f42017-10-13 21:26:13 +01004836 GEM_BUG_ON(i915_gem_object_has_pages(obj));
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004837
4838 if (obj->base.import_attach)
4839 drm_prime_gem_destroy(&obj->base, NULL);
4840
Chris Wilsond07f0e52016-10-28 13:58:44 +01004841 reservation_object_fini(&obj->__builtin_resv);
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004842 drm_gem_object_release(&obj->base);
4843 i915_gem_info_remove_obj(i915, obj->base.size);
4844
4845 kfree(obj->bit_17);
4846 i915_gem_object_free(obj);
Chris Wilsoncc731f52017-10-13 21:26:21 +01004847
Chris Wilsonc9c70472018-02-19 22:06:31 +00004848 GEM_BUG_ON(!atomic_read(&i915->mm.free_count));
4849 atomic_dec(&i915->mm.free_count);
4850
Chris Wilsoncc731f52017-10-13 21:26:21 +01004851 if (on)
4852 cond_resched();
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004853 }
Chris Wilsoncc731f52017-10-13 21:26:21 +01004854 intel_runtime_pm_put(i915);
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004855}
4856
4857static void i915_gem_flush_free_objects(struct drm_i915_private *i915)
4858{
4859 struct llist_node *freed;
4860
Chris Wilson87701b42017-10-13 21:26:20 +01004861 /* Free the oldest, most stale object to keep the free_list short */
4862 freed = NULL;
4863 if (!llist_empty(&i915->mm.free_list)) { /* quick test for hotpath */
4864 /* Only one consumer of llist_del_first() allowed */
4865 spin_lock(&i915->mm.free_lock);
4866 freed = llist_del_first(&i915->mm.free_list);
4867 spin_unlock(&i915->mm.free_lock);
4868 }
4869 if (unlikely(freed)) {
4870 freed->next = NULL;
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004871 __i915_gem_free_objects(i915, freed);
Chris Wilson87701b42017-10-13 21:26:20 +01004872 }
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004873}
4874
4875static void __i915_gem_free_work(struct work_struct *work)
4876{
4877 struct drm_i915_private *i915 =
4878 container_of(work, struct drm_i915_private, mm.free_work);
4879 struct llist_node *freed;
Chris Wilson26e12f82011-03-20 11:20:19 +00004880
Chris Wilson2ef1e722018-01-15 20:57:59 +00004881 /*
4882 * All file-owned VMA should have been released by this point through
Chris Wilsonb1f788c2016-08-04 07:52:45 +01004883 * i915_gem_close_object(), or earlier by i915_gem_context_close().
4884 * However, the object may also be bound into the global GTT (e.g.
4885 * older GPUs without per-process support, or for direct access through
4886 * the GTT either for the user or for scanout). Those VMA still need to
4887 * unbound now.
4888 */
Chris Wilson1488fc02012-04-24 15:47:31 +01004889
Chris Wilsonf991c492017-11-06 11:15:08 +00004890 spin_lock(&i915->mm.free_lock);
Chris Wilson5ad08be2017-04-07 11:25:51 +01004891 while ((freed = llist_del_all(&i915->mm.free_list))) {
Chris Wilsonf991c492017-11-06 11:15:08 +00004892 spin_unlock(&i915->mm.free_lock);
4893
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004894 __i915_gem_free_objects(i915, freed);
Chris Wilson5ad08be2017-04-07 11:25:51 +01004895 if (need_resched())
Chris Wilsonf991c492017-11-06 11:15:08 +00004896 return;
4897
4898 spin_lock(&i915->mm.free_lock);
Chris Wilson5ad08be2017-04-07 11:25:51 +01004899 }
Chris Wilsonf991c492017-11-06 11:15:08 +00004900 spin_unlock(&i915->mm.free_lock);
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004901}
4902
4903static void __i915_gem_free_object_rcu(struct rcu_head *head)
4904{
4905 struct drm_i915_gem_object *obj =
4906 container_of(head, typeof(*obj), rcu);
4907 struct drm_i915_private *i915 = to_i915(obj->base.dev);
4908
Chris Wilson2ef1e722018-01-15 20:57:59 +00004909 /*
4910 * Since we require blocking on struct_mutex to unbind the freed
4911 * object from the GPU before releasing resources back to the
4912 * system, we can not do that directly from the RCU callback (which may
4913 * be a softirq context), but must instead then defer that work onto a
4914 * kthread. We use the RCU callback rather than move the freed object
4915 * directly onto the work queue so that we can mix between using the
4916 * worker and performing frees directly from subsequent allocations for
4917 * crude but effective memory throttling.
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004918 */
4919 if (llist_add(&obj->freed, &i915->mm.free_list))
Chris Wilsonbeacbd12018-01-15 12:28:45 +00004920 queue_work(i915->wq, &i915->mm.free_work);
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004921}
4922
4923void i915_gem_free_object(struct drm_gem_object *gem_obj)
4924{
4925 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4926
Chris Wilsonbc0629a2016-11-01 10:03:17 +00004927 if (obj->mm.quirked)
4928 __i915_gem_object_unpin_pages(obj);
4929
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004930 if (discard_backing_storage(obj))
4931 obj->mm.madv = I915_MADV_DONTNEED;
Daniel Vettera071fa02014-06-18 23:28:09 +02004932
Chris Wilson2ef1e722018-01-15 20:57:59 +00004933 /*
4934 * Before we free the object, make sure any pure RCU-only
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004935 * read-side critical sections are complete, e.g.
4936 * i915_gem_busy_ioctl(). For the corresponding synchronized
4937 * lookup see i915_gem_object_lookup_rcu().
4938 */
Chris Wilsonc9c70472018-02-19 22:06:31 +00004939 atomic_inc(&to_i915(obj->base.dev)->mm.free_count);
Chris Wilsonfbbd37b2016-10-28 13:58:42 +01004940 call_rcu(&obj->rcu, __i915_gem_free_object_rcu);
Chris Wilsonbe726152010-07-23 23:18:50 +01004941}
4942
Chris Wilsonf8a7fde2016-10-28 13:58:29 +01004943void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
4944{
4945 lockdep_assert_held(&obj->base.dev->struct_mutex);
4946
Chris Wilsond1b48c12017-08-16 09:52:08 +01004947 if (!i915_gem_object_has_active_reference(obj) &&
4948 i915_gem_object_is_active(obj))
Chris Wilsonf8a7fde2016-10-28 13:58:29 +01004949 i915_gem_object_set_active_reference(obj);
4950 else
4951 i915_gem_object_put(obj);
4952}
4953
Chris Wilsonae6c4572017-11-10 14:26:28 +00004954static void assert_kernel_context_is_current(struct drm_i915_private *i915)
Chris Wilson3033aca2016-10-28 13:58:47 +01004955{
Chris Wilsonae6c4572017-11-10 14:26:28 +00004956 struct i915_gem_context *kernel_context = i915->kernel_context;
Chris Wilson3033aca2016-10-28 13:58:47 +01004957 struct intel_engine_cs *engine;
4958 enum intel_engine_id id;
4959
Chris Wilsonae6c4572017-11-10 14:26:28 +00004960 for_each_engine(engine, i915, id) {
4961 GEM_BUG_ON(__i915_gem_active_peek(&engine->timeline->last_request));
4962 GEM_BUG_ON(engine->last_retired_context != kernel_context);
4963 }
Chris Wilson3033aca2016-10-28 13:58:47 +01004964}
4965
Chris Wilson24145512017-01-24 11:01:35 +00004966void i915_gem_sanitize(struct drm_i915_private *i915)
4967{
Chris Wilsonf36325f2017-08-26 12:09:34 +01004968 if (i915_terminally_wedged(&i915->gpu_error)) {
4969 mutex_lock(&i915->drm.struct_mutex);
4970 i915_gem_unset_wedged(i915);
4971 mutex_unlock(&i915->drm.struct_mutex);
4972 }
4973
Chris Wilson24145512017-01-24 11:01:35 +00004974 /*
4975 * If we inherit context state from the BIOS or earlier occupants
4976 * of the GPU, the GPU may be in an inconsistent state when we
4977 * try to take over. The only way to remove the earlier state
4978 * is by resetting. However, resetting on earlier gen is tricky as
4979 * it may impact the display and we are uncertain about the stability
Joonas Lahtinenea117b82017-04-28 10:53:38 +03004980 * of the reset, so this could be applied to even earlier gen.
Chris Wilson24145512017-01-24 11:01:35 +00004981 */
Daniele Ceraolo Spurioce1599a2018-02-07 13:24:40 -08004982 if (INTEL_GEN(i915) >= 5 && intel_has_gpu_reset(i915))
4983 WARN_ON(intel_gpu_reset(i915, ALL_ENGINES));
Chris Wilson24145512017-01-24 11:01:35 +00004984}
4985
Tvrtko Ursulinbf9e8422016-12-01 14:16:38 +00004986int i915_gem_suspend(struct drm_i915_private *dev_priv)
Eric Anholt673a3942008-07-30 12:06:12 -07004987{
Tvrtko Ursulinbf9e8422016-12-01 14:16:38 +00004988 struct drm_device *dev = &dev_priv->drm;
Chris Wilsondcff85c2016-08-05 10:14:11 +01004989 int ret;
Eric Anholt673a3942008-07-30 12:06:12 -07004990
Chris Wilsonc998e8a2017-03-02 08:30:29 +00004991 intel_runtime_pm_get(dev_priv);
Chris Wilson54b4f682016-07-21 21:16:19 +01004992 intel_suspend_gt_powersave(dev_priv);
4993
Chris Wilson45c5f202013-10-16 11:50:01 +01004994 mutex_lock(&dev->struct_mutex);
Chris Wilson5ab57c72016-07-15 14:56:20 +01004995
4996 /* We have to flush all the executing contexts to main memory so
4997 * that they can saved in the hibernation image. To ensure the last
4998 * context image is coherent, we have to switch away from it. That
4999 * leaves the dev_priv->kernel_context still active when
5000 * we actually suspend, and its image in memory may not match the GPU
5001 * state. Fortunately, the kernel_context is disposable and we do
5002 * not rely on its state.
5003 */
Chris Wilsonecf73eb2017-11-30 10:29:51 +00005004 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
5005 ret = i915_gem_switch_to_kernel_context(dev_priv);
5006 if (ret)
5007 goto err_unlock;
Chris Wilson5ab57c72016-07-15 14:56:20 +01005008
Chris Wilsonecf73eb2017-11-30 10:29:51 +00005009 ret = i915_gem_wait_for_idle(dev_priv,
5010 I915_WAIT_INTERRUPTIBLE |
5011 I915_WAIT_LOCKED);
5012 if (ret && ret != -EIO)
5013 goto err_unlock;
Chris Wilsonf7403342013-09-13 23:57:04 +01005014
Chris Wilsonecf73eb2017-11-30 10:29:51 +00005015 assert_kernel_context_is_current(dev_priv);
5016 }
Chris Wilson829a0af2017-06-20 12:05:45 +01005017 i915_gem_contexts_lost(dev_priv);
Chris Wilson45c5f202013-10-16 11:50:01 +01005018 mutex_unlock(&dev->struct_mutex);
5019
Michal Wajdeczko7cfca4a2018-03-02 11:15:49 +00005020 intel_uc_suspend(dev_priv);
Sagar Arun Kamble63987bf2017-04-05 15:51:50 +05305021
Chris Wilson737b1502015-01-26 18:03:03 +02005022 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
Chris Wilson67d97da2016-07-04 08:08:31 +01005023 cancel_delayed_work_sync(&dev_priv->gt.retire_work);
Chris Wilsonbdeb9782016-12-23 14:57:56 +00005024
5025 /* As the idle_work is rearming if it detects a race, play safe and
5026 * repeat the flush until it is definitely idle.
5027 */
Chris Wilson7c262402017-10-06 11:40:38 +01005028 drain_delayed_work(&dev_priv->gt.idle_work);
Chris Wilsonbdeb9782016-12-23 14:57:56 +00005029
Chris Wilsonbdcf1202014-11-25 11:56:33 +00005030 /* Assert that we sucessfully flushed all the work and
5031 * reset the GPU back to its idle, low power state.
5032 */
Chris Wilson67d97da2016-07-04 08:08:31 +01005033 WARN_ON(dev_priv->gt.awake);
Chris Wilsonfc692bd2017-08-26 12:09:35 +01005034 if (WARN_ON(!intel_engines_are_idle(dev_priv)))
5035 i915_gem_set_wedged(dev_priv); /* no hope, discard everything */
Chris Wilsonbdcf1202014-11-25 11:56:33 +00005036
Imre Deak1c777c52016-10-12 17:46:37 +03005037 /*
5038 * Neither the BIOS, ourselves or any other kernel
5039 * expects the system to be in execlists mode on startup,
5040 * so we need to reset the GPU back to legacy mode. And the only
5041 * known way to disable logical contexts is through a GPU reset.
5042 *
5043 * So in order to leave the system in a known default configuration,
5044 * always reset the GPU upon unload and suspend. Afterwards we then
5045 * clean up the GEM state tracking, flushing off the requests and
5046 * leaving the system in a known idle state.
5047 *
5048 * Note that is of the upmost importance that the GPU is idle and
5049 * all stray writes are flushed *before* we dismantle the backing
5050 * storage for the pinned objects.
5051 *
5052 * However, since we are uncertain that resetting the GPU on older
5053 * machines is a good idea, we don't - just in case it leaves the
5054 * machine in an unusable condition.
5055 */
Michal Wajdeczkoc37d5722018-03-12 13:03:07 +00005056 intel_uc_sanitize(dev_priv);
Chris Wilson24145512017-01-24 11:01:35 +00005057 i915_gem_sanitize(dev_priv);
Chris Wilsoncad99462017-08-26 12:09:33 +01005058
5059 intel_runtime_pm_put(dev_priv);
5060 return 0;
Imre Deak1c777c52016-10-12 17:46:37 +03005061
Chris Wilsonc998e8a2017-03-02 08:30:29 +00005062err_unlock:
Chris Wilson45c5f202013-10-16 11:50:01 +01005063 mutex_unlock(&dev->struct_mutex);
Chris Wilsonc998e8a2017-03-02 08:30:29 +00005064 intel_runtime_pm_put(dev_priv);
Chris Wilson45c5f202013-10-16 11:50:01 +01005065 return ret;
Eric Anholt673a3942008-07-30 12:06:12 -07005066}
5067
Chris Wilson37cd3302017-11-12 11:27:38 +00005068void i915_gem_resume(struct drm_i915_private *i915)
Chris Wilson5ab57c72016-07-15 14:56:20 +01005069{
Chris Wilson37cd3302017-11-12 11:27:38 +00005070 WARN_ON(i915->gt.awake);
Chris Wilson5ab57c72016-07-15 14:56:20 +01005071
Chris Wilson37cd3302017-11-12 11:27:38 +00005072 mutex_lock(&i915->drm.struct_mutex);
5073 intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
Imre Deak31ab49a2016-11-07 11:20:05 +02005074
Chris Wilson37cd3302017-11-12 11:27:38 +00005075 i915_gem_restore_gtt_mappings(i915);
5076 i915_gem_restore_fences(i915);
Chris Wilson5ab57c72016-07-15 14:56:20 +01005077
Chris Wilson6ca9a2b2017-12-13 13:43:47 +00005078 /*
5079 * As we didn't flush the kernel context before suspend, we cannot
Chris Wilson5ab57c72016-07-15 14:56:20 +01005080 * guarantee that the context image is complete. So let's just reset
5081 * it and start again.
5082 */
Chris Wilson37cd3302017-11-12 11:27:38 +00005083 i915->gt.resume(i915);
Chris Wilson5ab57c72016-07-15 14:56:20 +01005084
Chris Wilson37cd3302017-11-12 11:27:38 +00005085 if (i915_gem_init_hw(i915))
5086 goto err_wedged;
5087
Michal Wajdeczko7cfca4a2018-03-02 11:15:49 +00005088 intel_uc_resume(i915);
Chris Wilson7469c622017-11-14 13:03:00 +00005089
Chris Wilson37cd3302017-11-12 11:27:38 +00005090 /* Always reload a context for powersaving. */
5091 if (i915_gem_switch_to_kernel_context(i915))
5092 goto err_wedged;
5093
5094out_unlock:
5095 intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
5096 mutex_unlock(&i915->drm.struct_mutex);
5097 return;
5098
5099err_wedged:
Chris Wilson6ca9a2b2017-12-13 13:43:47 +00005100 if (!i915_terminally_wedged(&i915->gpu_error)) {
5101 DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
5102 i915_gem_set_wedged(i915);
5103 }
Chris Wilson37cd3302017-11-12 11:27:38 +00005104 goto out_unlock;
Chris Wilson5ab57c72016-07-15 14:56:20 +01005105}
5106
Tvrtko Ursulinc6be6072016-11-16 08:55:31 +00005107void i915_gem_init_swizzling(struct drm_i915_private *dev_priv)
Daniel Vetterf691e2f2012-02-02 09:58:12 +01005108{
Tvrtko Ursulinc6be6072016-11-16 08:55:31 +00005109 if (INTEL_GEN(dev_priv) < 5 ||
Daniel Vetterf691e2f2012-02-02 09:58:12 +01005110 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
5111 return;
5112
5113 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
5114 DISP_TILE_SURFACE_SWIZZLING);
5115
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01005116 if (IS_GEN5(dev_priv))
Daniel Vetter11782b02012-01-31 16:47:55 +01005117 return;
5118
Daniel Vetterf691e2f2012-02-02 09:58:12 +01005119 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01005120 if (IS_GEN6(dev_priv))
Daniel Vetter6b26c862012-04-24 14:04:12 +02005121 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01005122 else if (IS_GEN7(dev_priv))
Daniel Vetter6b26c862012-04-24 14:04:12 +02005123 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
Tvrtko Ursulin5db94012016-10-13 11:03:10 +01005124 else if (IS_GEN8(dev_priv))
Ben Widawsky31a53362013-11-02 21:07:04 -07005125 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
Ben Widawsky8782e262012-12-18 10:31:23 -08005126 else
5127 BUG();
Daniel Vetterf691e2f2012-02-02 09:58:12 +01005128}
Daniel Vettere21af882012-02-09 20:53:27 +01005129
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +01005130static void init_unused_ring(struct drm_i915_private *dev_priv, u32 base)
Ville Syrjälä81e7f202014-08-15 01:21:55 +03005131{
Ville Syrjälä81e7f202014-08-15 01:21:55 +03005132 I915_WRITE(RING_CTL(base), 0);
5133 I915_WRITE(RING_HEAD(base), 0);
5134 I915_WRITE(RING_TAIL(base), 0);
5135 I915_WRITE(RING_START(base), 0);
5136}
5137
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +01005138static void init_unused_rings(struct drm_i915_private *dev_priv)
Ville Syrjälä81e7f202014-08-15 01:21:55 +03005139{
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +01005140 if (IS_I830(dev_priv)) {
5141 init_unused_ring(dev_priv, PRB1_BASE);
5142 init_unused_ring(dev_priv, SRB0_BASE);
5143 init_unused_ring(dev_priv, SRB1_BASE);
5144 init_unused_ring(dev_priv, SRB2_BASE);
5145 init_unused_ring(dev_priv, SRB3_BASE);
5146 } else if (IS_GEN2(dev_priv)) {
5147 init_unused_ring(dev_priv, SRB0_BASE);
5148 init_unused_ring(dev_priv, SRB1_BASE);
5149 } else if (IS_GEN3(dev_priv)) {
5150 init_unused_ring(dev_priv, PRB1_BASE);
5151 init_unused_ring(dev_priv, PRB2_BASE);
Ville Syrjälä81e7f202014-08-15 01:21:55 +03005152 }
5153}
5154
Chris Wilson20a8a742017-02-08 14:30:31 +00005155static int __i915_gem_restart_engines(void *data)
Ben Widawsky4fc7c972013-02-08 11:49:24 -08005156{
Chris Wilson20a8a742017-02-08 14:30:31 +00005157 struct drm_i915_private *i915 = data;
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00005158 struct intel_engine_cs *engine;
Akash Goel3b3f1652016-10-13 22:44:48 +05305159 enum intel_engine_id id;
Chris Wilson20a8a742017-02-08 14:30:31 +00005160 int err;
5161
5162 for_each_engine(engine, i915, id) {
5163 err = engine->init_hw(engine);
Chris Wilson8177e112018-02-07 11:15:45 +00005164 if (err) {
5165 DRM_ERROR("Failed to restart %s (%d)\n",
5166 engine->name, err);
Chris Wilson20a8a742017-02-08 14:30:31 +00005167 return err;
Chris Wilson8177e112018-02-07 11:15:45 +00005168 }
Chris Wilson20a8a742017-02-08 14:30:31 +00005169 }
5170
5171 return 0;
5172}
5173
5174int i915_gem_init_hw(struct drm_i915_private *dev_priv)
5175{
Chris Wilsond200cda2016-04-28 09:56:44 +01005176 int ret;
Ben Widawsky4fc7c972013-02-08 11:49:24 -08005177
Chris Wilsonde867c22016-10-25 13:16:02 +01005178 dev_priv->gt.last_init_time = ktime_get();
5179
Chris Wilson5e4f5182015-02-13 14:35:59 +00005180 /* Double layer security blanket, see i915_gem_init() */
5181 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5182
Tvrtko Ursulin0031fb92016-11-04 14:42:44 +00005183 if (HAS_EDRAM(dev_priv) && INTEL_GEN(dev_priv) < 9)
Ben Widawsky05e21cc2013-07-04 11:02:04 -07005184 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
Ben Widawsky4fc7c972013-02-08 11:49:24 -08005185
Tvrtko Ursulin772c2a52016-10-13 11:03:01 +01005186 if (IS_HASWELL(dev_priv))
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +01005187 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev_priv) ?
Ville Syrjälä0bf21342013-11-29 14:56:12 +02005188 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
Rodrigo Vivi94353732013-08-28 16:45:46 -03005189
Tvrtko Ursulin6e266952016-10-13 11:02:53 +01005190 if (HAS_PCH_NOP(dev_priv)) {
Tvrtko Ursulinfd6b8f42016-10-14 10:13:06 +01005191 if (IS_IVYBRIDGE(dev_priv)) {
Daniel Vetter6ba844b2014-01-22 23:39:30 +01005192 u32 temp = I915_READ(GEN7_MSG_CTL);
5193 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
5194 I915_WRITE(GEN7_MSG_CTL, temp);
Tvrtko Ursulinc6be6072016-11-16 08:55:31 +00005195 } else if (INTEL_GEN(dev_priv) >= 7) {
Daniel Vetter6ba844b2014-01-22 23:39:30 +01005196 u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
5197 temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
5198 I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
5199 }
Ben Widawsky88a2b2a2013-04-05 13:12:43 -07005200 }
5201
Tvrtko Ursulinc6be6072016-11-16 08:55:31 +00005202 i915_gem_init_swizzling(dev_priv);
Ben Widawsky4fc7c972013-02-08 11:49:24 -08005203
Daniel Vetterd5abdfd2014-11-20 09:45:19 +01005204 /*
5205 * At least 830 can leave some of the unused rings
5206 * "active" (ie. head != tail) after resume which
5207 * will prevent c3 entry. Makes sure all unused rings
5208 * are totally idle.
5209 */
Tvrtko Ursulin50a0bc92016-10-13 11:02:58 +01005210 init_unused_rings(dev_priv);
Daniel Vetterd5abdfd2014-11-20 09:45:19 +01005211
Dave Gordoned54c1a2016-01-19 19:02:54 +00005212 BUG_ON(!dev_priv->kernel_context);
Chris Wilson6f74b362017-10-15 15:37:25 +01005213 if (i915_terminally_wedged(&dev_priv->gpu_error)) {
5214 ret = -EIO;
5215 goto out;
5216 }
John Harrison90638cc2015-05-29 17:43:37 +01005217
Tvrtko Ursulinc6be6072016-11-16 08:55:31 +00005218 ret = i915_ppgtt_init_hw(dev_priv);
John Harrison4ad2fd82015-06-18 13:11:20 +01005219 if (ret) {
Chris Wilson8177e112018-02-07 11:15:45 +00005220 DRM_ERROR("Enabling PPGTT failed (%d)\n", ret);
John Harrison4ad2fd82015-06-18 13:11:20 +01005221 goto out;
5222 }
5223
Jackie Lif08e2032018-03-13 17:32:53 -07005224 ret = intel_wopcm_init_hw(&dev_priv->wopcm);
5225 if (ret) {
5226 DRM_ERROR("Enabling WOPCM failed (%d)\n", ret);
5227 goto out;
5228 }
5229
Michał Winiarski9bdc3572017-10-25 18:25:19 +01005230 /* We can't enable contexts until all firmware is loaded */
5231 ret = intel_uc_init_hw(dev_priv);
Chris Wilson8177e112018-02-07 11:15:45 +00005232 if (ret) {
5233 DRM_ERROR("Enabling uc failed (%d)\n", ret);
Michał Winiarski9bdc3572017-10-25 18:25:19 +01005234 goto out;
Chris Wilson8177e112018-02-07 11:15:45 +00005235 }
Michał Winiarski9bdc3572017-10-25 18:25:19 +01005236
Tvrtko Ursulinbf9e8422016-12-01 14:16:38 +00005237 intel_mocs_init_l3cc_table(dev_priv);
Peter Antoine0ccdacf2016-04-13 15:03:25 +01005238
Chris Wilson136109c2017-11-02 13:14:30 +00005239 /* Only when the HW is re-initialised, can we replay the requests */
5240 ret = __i915_gem_restart_engines(dev_priv);
Chris Wilson5e4f5182015-02-13 14:35:59 +00005241out:
5242 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
Ben Widawsky2fa48d82013-12-06 14:11:04 -08005243 return ret;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08005244}
5245
Chris Wilsond2b4b972017-11-10 14:26:33 +00005246static int __intel_engines_record_defaults(struct drm_i915_private *i915)
5247{
5248 struct i915_gem_context *ctx;
5249 struct intel_engine_cs *engine;
5250 enum intel_engine_id id;
5251 int err;
5252
5253 /*
5254 * As we reset the gpu during very early sanitisation, the current
5255 * register state on the GPU should reflect its defaults values.
5256 * We load a context onto the hw (with restore-inhibit), then switch
5257 * over to a second context to save that default register state. We
5258 * can then prime every new context with that state so they all start
5259 * from the same default HW values.
5260 */
5261
5262 ctx = i915_gem_context_create_kernel(i915, 0);
5263 if (IS_ERR(ctx))
5264 return PTR_ERR(ctx);
5265
5266 for_each_engine(engine, i915, id) {
Chris Wilsone61e0f52018-02-21 09:56:36 +00005267 struct i915_request *rq;
Chris Wilsond2b4b972017-11-10 14:26:33 +00005268
Chris Wilsone61e0f52018-02-21 09:56:36 +00005269 rq = i915_request_alloc(engine, ctx);
Chris Wilsond2b4b972017-11-10 14:26:33 +00005270 if (IS_ERR(rq)) {
5271 err = PTR_ERR(rq);
5272 goto out_ctx;
5273 }
5274
Chris Wilson3fef5cd2017-11-20 10:20:02 +00005275 err = 0;
Chris Wilsond2b4b972017-11-10 14:26:33 +00005276 if (engine->init_context)
5277 err = engine->init_context(rq);
5278
Chris Wilsone61e0f52018-02-21 09:56:36 +00005279 __i915_request_add(rq, true);
Chris Wilsond2b4b972017-11-10 14:26:33 +00005280 if (err)
5281 goto err_active;
5282 }
5283
5284 err = i915_gem_switch_to_kernel_context(i915);
5285 if (err)
5286 goto err_active;
5287
5288 err = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
5289 if (err)
5290 goto err_active;
5291
5292 assert_kernel_context_is_current(i915);
5293
5294 for_each_engine(engine, i915, id) {
5295 struct i915_vma *state;
5296
5297 state = ctx->engine[id].state;
5298 if (!state)
5299 continue;
5300
5301 /*
5302 * As we will hold a reference to the logical state, it will
5303 * not be torn down with the context, and importantly the
5304 * object will hold onto its vma (making it possible for a
5305 * stray GTT write to corrupt our defaults). Unmap the vma
5306 * from the GTT to prevent such accidents and reclaim the
5307 * space.
5308 */
5309 err = i915_vma_unbind(state);
5310 if (err)
5311 goto err_active;
5312
5313 err = i915_gem_object_set_to_cpu_domain(state->obj, false);
5314 if (err)
5315 goto err_active;
5316
5317 engine->default_state = i915_gem_object_get(state->obj);
5318 }
5319
5320 if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) {
5321 unsigned int found = intel_engines_has_context_isolation(i915);
5322
5323 /*
5324 * Make sure that classes with multiple engine instances all
5325 * share the same basic configuration.
5326 */
5327 for_each_engine(engine, i915, id) {
5328 unsigned int bit = BIT(engine->uabi_class);
5329 unsigned int expected = engine->default_state ? bit : 0;
5330
5331 if ((found & bit) != expected) {
5332 DRM_ERROR("mismatching default context state for class %d on engine %s\n",
5333 engine->uabi_class, engine->name);
5334 }
5335 }
5336 }
5337
5338out_ctx:
5339 i915_gem_context_set_closed(ctx);
5340 i915_gem_context_put(ctx);
5341 return err;
5342
5343err_active:
5344 /*
5345 * If we have to abandon now, we expect the engines to be idle
5346 * and ready to be torn-down. First try to flush any remaining
5347 * request, ensure we are pointing at the kernel context and
5348 * then remove it.
5349 */
5350 if (WARN_ON(i915_gem_switch_to_kernel_context(i915)))
5351 goto out_ctx;
5352
5353 if (WARN_ON(i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED)))
5354 goto out_ctx;
5355
5356 i915_gem_contexts_lost(i915);
5357 goto out_ctx;
5358}
5359
Tvrtko Ursulinbf9e8422016-12-01 14:16:38 +00005360int i915_gem_init(struct drm_i915_private *dev_priv)
Chris Wilson1070a422012-04-24 15:47:41 +01005361{
Chris Wilson1070a422012-04-24 15:47:41 +01005362 int ret;
5363
Matthew Auldda9fe3f32017-10-06 23:18:31 +01005364 /*
5365 * We need to fallback to 4K pages since gvt gtt handling doesn't
5366 * support huge page entries - we will need to check either hypervisor
5367 * mm can support huge guest page or just do emulation in gvt.
5368 */
5369 if (intel_vgpu_active(dev_priv))
5370 mkwrite_device_info(dev_priv)->page_sizes =
5371 I915_GTT_PAGE_SIZE_4K;
5372
Chris Wilson94312822017-05-03 10:39:18 +01005373 dev_priv->mm.unordered_timeline = dma_fence_context_alloc(1);
Chris Wilson57822dc2017-02-22 11:40:48 +00005374
Chris Wilsonfb5c5512017-11-20 20:55:00 +00005375 if (HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
Chris Wilson821ed7d2016-09-09 14:11:53 +01005376 dev_priv->gt.resume = intel_lr_context_resume;
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00005377 dev_priv->gt.cleanup_engine = intel_logical_ring_cleanup;
Chris Wilsonfb5c5512017-11-20 20:55:00 +00005378 } else {
5379 dev_priv->gt.resume = intel_legacy_submission_resume;
5380 dev_priv->gt.cleanup_engine = intel_engine_cleanup;
Oscar Mateoa83014d2014-07-24 17:04:21 +01005381 }
5382
Chris Wilsonee487002017-11-22 17:26:21 +00005383 ret = i915_gem_init_userptr(dev_priv);
5384 if (ret)
5385 return ret;
5386
Jackie Li6b0478f2018-03-13 17:32:50 -07005387 ret = intel_wopcm_init(&dev_priv->wopcm);
5388 if (ret)
5389 return ret;
5390
Sagar Arun Kamble70deead2018-01-24 21:16:58 +05305391 ret = intel_uc_init_misc(dev_priv);
Michał Winiarski3176ff42017-12-13 23:13:47 +01005392 if (ret)
5393 return ret;
5394
Chris Wilson5e4f5182015-02-13 14:35:59 +00005395 /* This is just a security blanket to placate dragons.
5396 * On some systems, we very sporadically observe that the first TLBs
5397 * used by the CS may be stale, despite us poking the TLB reset. If
5398 * we hold the forcewake during initialisation these problems
5399 * just magically go away.
5400 */
Chris Wilsonee487002017-11-22 17:26:21 +00005401 mutex_lock(&dev_priv->drm.struct_mutex);
Chris Wilson5e4f5182015-02-13 14:35:59 +00005402 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
5403
Chris Wilsonf6b9d5c2016-08-04 07:52:23 +01005404 ret = i915_gem_init_ggtt(dev_priv);
Chris Wilson6ca9a2b2017-12-13 13:43:47 +00005405 if (ret) {
5406 GEM_BUG_ON(ret == -EIO);
5407 goto err_unlock;
5408 }
Jesse Barnesd62b4892013-03-08 10:45:53 -08005409
Chris Wilson829a0af2017-06-20 12:05:45 +01005410 ret = i915_gem_contexts_init(dev_priv);
Chris Wilson6ca9a2b2017-12-13 13:43:47 +00005411 if (ret) {
5412 GEM_BUG_ON(ret == -EIO);
5413 goto err_ggtt;
5414 }
Ben Widawsky2fa48d82013-12-06 14:11:04 -08005415
Tvrtko Ursulinbf9e8422016-12-01 14:16:38 +00005416 ret = intel_engines_init(dev_priv);
Chris Wilson6ca9a2b2017-12-13 13:43:47 +00005417 if (ret) {
5418 GEM_BUG_ON(ret == -EIO);
5419 goto err_context;
5420 }
Daniel Vetter53ca26c2012-04-26 23:28:03 +02005421
Chris Wilsonf58d13d2017-11-10 14:26:29 +00005422 intel_init_gt_powersave(dev_priv);
5423
Michał Winiarski61b5c152017-12-13 23:13:48 +01005424 ret = intel_uc_init(dev_priv);
Chris Wilsoncc6a8182017-11-10 14:26:30 +00005425 if (ret)
Chris Wilson6ca9a2b2017-12-13 13:43:47 +00005426 goto err_pm;
Chris Wilsoncc6a8182017-11-10 14:26:30 +00005427
Michał Winiarski61b5c152017-12-13 23:13:48 +01005428 ret = i915_gem_init_hw(dev_priv);
5429 if (ret)
5430 goto err_uc_init;
5431
Chris Wilsoncc6a8182017-11-10 14:26:30 +00005432 /*
5433 * Despite its name intel_init_clock_gating applies both display
5434 * clock gating workarounds; GT mmio workarounds and the occasional
5435 * GT power context workaround. Worse, sometimes it includes a context
5436 * register workaround which we need to apply before we record the
5437 * default HW state for all contexts.
5438 *
5439 * FIXME: break up the workarounds and apply them at the right time!
5440 */
5441 intel_init_clock_gating(dev_priv);
5442
Chris Wilsond2b4b972017-11-10 14:26:33 +00005443 ret = __intel_engines_record_defaults(dev_priv);
Chris Wilson6ca9a2b2017-12-13 13:43:47 +00005444 if (ret)
5445 goto err_init_hw;
5446
5447 if (i915_inject_load_failure()) {
5448 ret = -ENODEV;
5449 goto err_init_hw;
5450 }
5451
5452 if (i915_inject_load_failure()) {
5453 ret = -EIO;
5454 goto err_init_hw;
5455 }
5456
5457 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5458 mutex_unlock(&dev_priv->drm.struct_mutex);
5459
5460 return 0;
5461
5462 /*
5463 * Unwinding is complicated by that we want to handle -EIO to mean
5464 * disable GPU submission but keep KMS alive. We want to mark the
5465 * HW as irrevisibly wedged, but keep enough state around that the
5466 * driver doesn't explode during runtime.
5467 */
5468err_init_hw:
5469 i915_gem_wait_for_idle(dev_priv, I915_WAIT_LOCKED);
5470 i915_gem_contexts_lost(dev_priv);
5471 intel_uc_fini_hw(dev_priv);
Michał Winiarski61b5c152017-12-13 23:13:48 +01005472err_uc_init:
5473 intel_uc_fini(dev_priv);
Chris Wilson6ca9a2b2017-12-13 13:43:47 +00005474err_pm:
5475 if (ret != -EIO) {
5476 intel_cleanup_gt_powersave(dev_priv);
5477 i915_gem_cleanup_engines(dev_priv);
5478 }
5479err_context:
5480 if (ret != -EIO)
5481 i915_gem_contexts_fini(dev_priv);
5482err_ggtt:
5483err_unlock:
5484 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5485 mutex_unlock(&dev_priv->drm.struct_mutex);
5486
Sagar Arun Kamble70deead2018-01-24 21:16:58 +05305487 intel_uc_fini_misc(dev_priv);
Sagar Arun Kambleda943b52018-01-10 18:24:16 +05305488
Chris Wilson6ca9a2b2017-12-13 13:43:47 +00005489 if (ret != -EIO)
5490 i915_gem_cleanup_userptr(dev_priv);
5491
Chris Wilson60990322014-04-09 09:19:42 +01005492 if (ret == -EIO) {
Chris Wilson6ca9a2b2017-12-13 13:43:47 +00005493 /*
5494 * Allow engine initialisation to fail by marking the GPU as
Chris Wilson60990322014-04-09 09:19:42 +01005495 * wedged. But we only want to do this where the GPU is angry,
5496 * for all other failure, such as an allocation failure, bail.
5497 */
Chris Wilson6f74b362017-10-15 15:37:25 +01005498 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
5499 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
5500 i915_gem_set_wedged(dev_priv);
5501 }
Chris Wilson60990322014-04-09 09:19:42 +01005502 ret = 0;
Chris Wilson1070a422012-04-24 15:47:41 +01005503 }
5504
Chris Wilson6ca9a2b2017-12-13 13:43:47 +00005505 i915_gem_drain_freed_objects(dev_priv);
Chris Wilson60990322014-04-09 09:19:42 +01005506 return ret;
Chris Wilson1070a422012-04-24 15:47:41 +01005507}
5508
Chris Wilson24145512017-01-24 11:01:35 +00005509void i915_gem_init_mmio(struct drm_i915_private *i915)
5510{
5511 i915_gem_sanitize(i915);
5512}
5513
Zou Nan hai8187a2b2010-05-21 09:08:55 +08005514void
Tvrtko Ursulincb15d9f2016-12-01 14:16:39 +00005515i915_gem_cleanup_engines(struct drm_i915_private *dev_priv)
Zou Nan hai8187a2b2010-05-21 09:08:55 +08005516{
Tvrtko Ursuline2f80392016-03-16 11:00:36 +00005517 struct intel_engine_cs *engine;
Akash Goel3b3f1652016-10-13 22:44:48 +05305518 enum intel_engine_id id;
Zou Nan hai8187a2b2010-05-21 09:08:55 +08005519
Akash Goel3b3f1652016-10-13 22:44:48 +05305520 for_each_engine(engine, dev_priv, id)
Tvrtko Ursulin117897f2016-03-16 11:00:40 +00005521 dev_priv->gt.cleanup_engine(engine);
Zou Nan hai8187a2b2010-05-21 09:08:55 +08005522}
5523
Eric Anholt673a3942008-07-30 12:06:12 -07005524void
Imre Deak40ae4e12016-03-16 14:54:03 +02005525i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
5526{
Chris Wilson49ef5292016-08-18 17:17:00 +01005527 int i;
Imre Deak40ae4e12016-03-16 14:54:03 +02005528
Tvrtko Ursulinc56b89f2018-02-09 21:58:46 +00005529 if (INTEL_GEN(dev_priv) >= 7 && !IS_VALLEYVIEW(dev_priv) &&
Imre Deak40ae4e12016-03-16 14:54:03 +02005530 !IS_CHERRYVIEW(dev_priv))
5531 dev_priv->num_fence_regs = 32;
Tvrtko Ursulinc56b89f2018-02-09 21:58:46 +00005532 else if (INTEL_GEN(dev_priv) >= 4 ||
Jani Nikula73f67aa2016-12-07 22:48:09 +02005533 IS_I945G(dev_priv) || IS_I945GM(dev_priv) ||
5534 IS_G33(dev_priv) || IS_PINEVIEW(dev_priv))
Imre Deak40ae4e12016-03-16 14:54:03 +02005535 dev_priv->num_fence_regs = 16;
5536 else
5537 dev_priv->num_fence_regs = 8;
5538
Chris Wilsonc0336662016-05-06 15:40:21 +01005539 if (intel_vgpu_active(dev_priv))
Imre Deak40ae4e12016-03-16 14:54:03 +02005540 dev_priv->num_fence_regs =
5541 I915_READ(vgtif_reg(avail_rs.fence_num));
5542
5543 /* Initialize fence registers to zero */
Chris Wilson49ef5292016-08-18 17:17:00 +01005544 for (i = 0; i < dev_priv->num_fence_regs; i++) {
5545 struct drm_i915_fence_reg *fence = &dev_priv->fence_regs[i];
5546
5547 fence->i915 = dev_priv;
5548 fence->id = i;
5549 list_add_tail(&fence->link, &dev_priv->mm.fence_list);
5550 }
Tvrtko Ursulin4362f4f2016-11-16 08:55:33 +00005551 i915_gem_restore_fences(dev_priv);
Imre Deak40ae4e12016-03-16 14:54:03 +02005552
Tvrtko Ursulin4362f4f2016-11-16 08:55:33 +00005553 i915_gem_detect_bit_6_swizzle(dev_priv);
Imre Deak40ae4e12016-03-16 14:54:03 +02005554}
5555
Chris Wilson9c52d1c2017-11-10 23:24:47 +00005556static void i915_gem_init__mm(struct drm_i915_private *i915)
5557{
5558 spin_lock_init(&i915->mm.object_stat_lock);
5559 spin_lock_init(&i915->mm.obj_lock);
5560 spin_lock_init(&i915->mm.free_lock);
5561
5562 init_llist_head(&i915->mm.free_list);
5563
5564 INIT_LIST_HEAD(&i915->mm.unbound_list);
5565 INIT_LIST_HEAD(&i915->mm.bound_list);
5566 INIT_LIST_HEAD(&i915->mm.fence_list);
5567 INIT_LIST_HEAD(&i915->mm.userfault_list);
5568
5569 INIT_WORK(&i915->mm.free_work, __i915_gem_free_work);
5570}
5571
Michal Wajdeczkoa0de9082018-03-23 12:34:49 +00005572int i915_gem_init_early(struct drm_i915_private *dev_priv)
Eric Anholt673a3942008-07-30 12:06:12 -07005573{
Tvrtko Ursulina9335682016-11-02 15:14:59 +00005574 int err = -ENOMEM;
Chris Wilson42dcedd2012-11-15 11:32:30 +00005575
Tvrtko Ursulina9335682016-11-02 15:14:59 +00005576 dev_priv->objects = KMEM_CACHE(drm_i915_gem_object, SLAB_HWCACHE_ALIGN);
5577 if (!dev_priv->objects)
Chris Wilson73cb9702016-10-28 13:58:46 +01005578 goto err_out;
Chris Wilson73cb9702016-10-28 13:58:46 +01005579
Tvrtko Ursulina9335682016-11-02 15:14:59 +00005580 dev_priv->vmas = KMEM_CACHE(i915_vma, SLAB_HWCACHE_ALIGN);
5581 if (!dev_priv->vmas)
Chris Wilson73cb9702016-10-28 13:58:46 +01005582 goto err_objects;
Chris Wilson73cb9702016-10-28 13:58:46 +01005583
Chris Wilsond1b48c12017-08-16 09:52:08 +01005584 dev_priv->luts = KMEM_CACHE(i915_lut_handle, 0);
5585 if (!dev_priv->luts)
5586 goto err_vmas;
5587
Chris Wilsone61e0f52018-02-21 09:56:36 +00005588 dev_priv->requests = KMEM_CACHE(i915_request,
Tvrtko Ursulina9335682016-11-02 15:14:59 +00005589 SLAB_HWCACHE_ALIGN |
5590 SLAB_RECLAIM_ACCOUNT |
Paul E. McKenney5f0d5a32017-01-18 02:53:44 -08005591 SLAB_TYPESAFE_BY_RCU);
Tvrtko Ursulina9335682016-11-02 15:14:59 +00005592 if (!dev_priv->requests)
Chris Wilsond1b48c12017-08-16 09:52:08 +01005593 goto err_luts;
Chris Wilson73cb9702016-10-28 13:58:46 +01005594
Chris Wilson52e54202016-11-14 20:41:02 +00005595 dev_priv->dependencies = KMEM_CACHE(i915_dependency,
5596 SLAB_HWCACHE_ALIGN |
5597 SLAB_RECLAIM_ACCOUNT);
5598 if (!dev_priv->dependencies)
5599 goto err_requests;
5600
Chris Wilsonc5cf9a92017-05-17 13:10:04 +01005601 dev_priv->priorities = KMEM_CACHE(i915_priolist, SLAB_HWCACHE_ALIGN);
5602 if (!dev_priv->priorities)
5603 goto err_dependencies;
5604
Chris Wilson73cb9702016-10-28 13:58:46 +01005605 mutex_lock(&dev_priv->drm.struct_mutex);
5606 INIT_LIST_HEAD(&dev_priv->gt.timelines);
Chris Wilsonbb894852016-11-14 20:40:57 +00005607 err = i915_gem_timeline_init__global(dev_priv);
Chris Wilson73cb9702016-10-28 13:58:46 +01005608 mutex_unlock(&dev_priv->drm.struct_mutex);
5609 if (err)
Chris Wilsonc5cf9a92017-05-17 13:10:04 +01005610 goto err_priorities;
Eric Anholt673a3942008-07-30 12:06:12 -07005611
Chris Wilson9c52d1c2017-11-10 23:24:47 +00005612 i915_gem_init__mm(dev_priv);
Chris Wilsonf2123812017-10-16 12:40:37 +01005613
Chris Wilson67d97da2016-07-04 08:08:31 +01005614 INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
Eric Anholt673a3942008-07-30 12:06:12 -07005615 i915_gem_retire_work_handler);
Chris Wilson67d97da2016-07-04 08:08:31 +01005616 INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005617 i915_gem_idle_work_handler);
Chris Wilson1f15b762016-07-01 17:23:14 +01005618 init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
Daniel Vetter1f83fee2012-11-15 17:17:22 +01005619 init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
Chris Wilson31169712009-09-14 16:50:28 +01005620
Joonas Lahtinen6f633402016-09-01 14:58:21 +03005621 atomic_set(&dev_priv->mm.bsd_engine_dispatch_index, 0);
5622
Chris Wilsonb5add952016-08-04 16:32:36 +01005623 spin_lock_init(&dev_priv->fb_tracking.lock);
Chris Wilson73cb9702016-10-28 13:58:46 +01005624
Matthew Auld465c4032017-10-06 23:18:14 +01005625 err = i915_gemfs_init(dev_priv);
5626 if (err)
5627 DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", err);
5628
Chris Wilson73cb9702016-10-28 13:58:46 +01005629 return 0;
5630
Chris Wilsonc5cf9a92017-05-17 13:10:04 +01005631err_priorities:
5632 kmem_cache_destroy(dev_priv->priorities);
Chris Wilson52e54202016-11-14 20:41:02 +00005633err_dependencies:
5634 kmem_cache_destroy(dev_priv->dependencies);
Chris Wilson73cb9702016-10-28 13:58:46 +01005635err_requests:
5636 kmem_cache_destroy(dev_priv->requests);
Chris Wilsond1b48c12017-08-16 09:52:08 +01005637err_luts:
5638 kmem_cache_destroy(dev_priv->luts);
Chris Wilson73cb9702016-10-28 13:58:46 +01005639err_vmas:
5640 kmem_cache_destroy(dev_priv->vmas);
5641err_objects:
5642 kmem_cache_destroy(dev_priv->objects);
5643err_out:
5644 return err;
Eric Anholt673a3942008-07-30 12:06:12 -07005645}
Dave Airlie71acb5e2008-12-30 20:31:46 +10005646
Michal Wajdeczkoa0de9082018-03-23 12:34:49 +00005647void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
Imre Deakd64aa092016-01-19 15:26:29 +02005648{
Chris Wilsonc4d4c1c2017-02-10 16:35:23 +00005649 i915_gem_drain_freed_objects(dev_priv);
Chris Wilsonc9c70472018-02-19 22:06:31 +00005650 GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
5651 GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
Chris Wilsonc4d4c1c2017-02-10 16:35:23 +00005652 WARN_ON(dev_priv->mm.object_count);
Chris Wilson7d5d59e2016-11-01 08:48:41 +00005653
Matthew Auldea84aa72016-11-17 21:04:11 +00005654 mutex_lock(&dev_priv->drm.struct_mutex);
5655 i915_gem_timeline_fini(&dev_priv->gt.global_timeline);
5656 WARN_ON(!list_empty(&dev_priv->gt.timelines));
5657 mutex_unlock(&dev_priv->drm.struct_mutex);
5658
Chris Wilsonc5cf9a92017-05-17 13:10:04 +01005659 kmem_cache_destroy(dev_priv->priorities);
Chris Wilson52e54202016-11-14 20:41:02 +00005660 kmem_cache_destroy(dev_priv->dependencies);
Imre Deakd64aa092016-01-19 15:26:29 +02005661 kmem_cache_destroy(dev_priv->requests);
Chris Wilsond1b48c12017-08-16 09:52:08 +01005662 kmem_cache_destroy(dev_priv->luts);
Imre Deakd64aa092016-01-19 15:26:29 +02005663 kmem_cache_destroy(dev_priv->vmas);
5664 kmem_cache_destroy(dev_priv->objects);
Chris Wilson0eafec62016-08-04 16:32:41 +01005665
5666 /* And ensure that our DESTROY_BY_RCU slabs are truly destroyed */
5667 rcu_barrier();
Matthew Auld465c4032017-10-06 23:18:14 +01005668
5669 i915_gemfs_fini(dev_priv);
Imre Deakd64aa092016-01-19 15:26:29 +02005670}
5671
Chris Wilson6a800ea2016-09-21 14:51:07 +01005672int i915_gem_freeze(struct drm_i915_private *dev_priv)
5673{
Chris Wilsond0aa3012017-04-07 11:25:49 +01005674 /* Discard all purgeable objects, let userspace recover those as
5675 * required after resuming.
5676 */
Chris Wilson6a800ea2016-09-21 14:51:07 +01005677 i915_gem_shrink_all(dev_priv);
Chris Wilson6a800ea2016-09-21 14:51:07 +01005678
Chris Wilson6a800ea2016-09-21 14:51:07 +01005679 return 0;
5680}
5681
Chris Wilson461fb992016-05-14 07:26:33 +01005682int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
5683{
5684 struct drm_i915_gem_object *obj;
Chris Wilson7aab2d52016-09-09 20:02:18 +01005685 struct list_head *phases[] = {
5686 &dev_priv->mm.unbound_list,
5687 &dev_priv->mm.bound_list,
5688 NULL
5689 }, **p;
Chris Wilson461fb992016-05-14 07:26:33 +01005690
5691 /* Called just before we write the hibernation image.
5692 *
5693 * We need to update the domain tracking to reflect that the CPU
5694 * will be accessing all the pages to create and restore from the
5695 * hibernation, and so upon restoration those pages will be in the
5696 * CPU domain.
5697 *
5698 * To make sure the hibernation image contains the latest state,
5699 * we update that state just before writing out the image.
Chris Wilson7aab2d52016-09-09 20:02:18 +01005700 *
5701 * To try and reduce the hibernation image, we manually shrink
Chris Wilsond0aa3012017-04-07 11:25:49 +01005702 * the objects as well, see i915_gem_freeze()
Chris Wilson461fb992016-05-14 07:26:33 +01005703 */
5704
Chris Wilson912d5722017-09-06 16:19:30 -07005705 i915_gem_shrink(dev_priv, -1UL, NULL, I915_SHRINK_UNBOUND);
Chris Wilson17b93c42017-04-07 11:25:50 +01005706 i915_gem_drain_freed_objects(dev_priv);
Chris Wilson461fb992016-05-14 07:26:33 +01005707
Chris Wilsonf2123812017-10-16 12:40:37 +01005708 spin_lock(&dev_priv->mm.obj_lock);
Chris Wilson7aab2d52016-09-09 20:02:18 +01005709 for (p = phases; *p; p++) {
Chris Wilsonf2123812017-10-16 12:40:37 +01005710 list_for_each_entry(obj, *p, mm.link)
Chris Wilsone27ab732017-06-15 13:38:49 +01005711 __start_cpu_write(obj);
Chris Wilson461fb992016-05-14 07:26:33 +01005712 }
Chris Wilsonf2123812017-10-16 12:40:37 +01005713 spin_unlock(&dev_priv->mm.obj_lock);
Chris Wilson461fb992016-05-14 07:26:33 +01005714
5715 return 0;
5716}
5717
Chris Wilsonf787a5f2010-09-24 16:02:42 +01005718void i915_gem_release(struct drm_device *dev, struct drm_file *file)
Eric Anholtb9624422009-06-03 07:27:35 +00005719{
Chris Wilsonf787a5f2010-09-24 16:02:42 +01005720 struct drm_i915_file_private *file_priv = file->driver_priv;
Chris Wilsone61e0f52018-02-21 09:56:36 +00005721 struct i915_request *request;
Eric Anholtb9624422009-06-03 07:27:35 +00005722
5723 /* Clean up our request list when the client is going away, so that
5724 * later retire_requests won't dereference our soon-to-be-gone
5725 * file_priv.
5726 */
Chris Wilson1c255952010-09-26 11:03:27 +01005727 spin_lock(&file_priv->mm.lock);
Chris Wilsonc8659ef2017-03-02 12:25:25 +00005728 list_for_each_entry(request, &file_priv->mm.request_list, client_link)
Chris Wilsonf787a5f2010-09-24 16:02:42 +01005729 request->file_priv = NULL;
Chris Wilson1c255952010-09-26 11:03:27 +01005730 spin_unlock(&file_priv->mm.lock);
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005731}
5732
Chris Wilson829a0af2017-06-20 12:05:45 +01005733int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005734{
5735 struct drm_i915_file_private *file_priv;
Ben Widawskye422b882013-12-06 14:10:58 -08005736 int ret;
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005737
Chris Wilsonc4c29d72016-11-09 10:45:07 +00005738 DRM_DEBUG("\n");
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005739
5740 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
5741 if (!file_priv)
5742 return -ENOMEM;
5743
5744 file->driver_priv = file_priv;
Chris Wilson829a0af2017-06-20 12:05:45 +01005745 file_priv->dev_priv = i915;
Chris Wilsonab0e7ff2014-02-25 17:11:24 +02005746 file_priv->file = file;
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005747
5748 spin_lock_init(&file_priv->mm.lock);
5749 INIT_LIST_HEAD(&file_priv->mm.request_list);
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005750
Chris Wilsonc80ff162016-07-27 09:07:27 +01005751 file_priv->bsd_engine = -1;
Tvrtko Ursulinde1add32016-01-15 15:12:50 +00005752
Chris Wilson829a0af2017-06-20 12:05:45 +01005753 ret = i915_gem_context_open(i915, file);
Ben Widawskye422b882013-12-06 14:10:58 -08005754 if (ret)
5755 kfree(file_priv);
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005756
Ben Widawskye422b882013-12-06 14:10:58 -08005757 return ret;
Chris Wilsonb29c19b2013-09-25 17:34:56 +01005758}
5759
Daniel Vetterb680c372014-09-19 18:27:27 +02005760/**
5761 * i915_gem_track_fb - update frontbuffer tracking
Geliang Tangd9072a32015-09-15 05:58:44 -07005762 * @old: current GEM buffer for the frontbuffer slots
5763 * @new: new GEM buffer for the frontbuffer slots
5764 * @frontbuffer_bits: bitmask of frontbuffer slots
Daniel Vetterb680c372014-09-19 18:27:27 +02005765 *
5766 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
5767 * from @old and setting them in @new. Both @old and @new can be NULL.
5768 */
Daniel Vettera071fa02014-06-18 23:28:09 +02005769void i915_gem_track_fb(struct drm_i915_gem_object *old,
5770 struct drm_i915_gem_object *new,
5771 unsigned frontbuffer_bits)
5772{
Chris Wilsonfaf5bf02016-08-04 16:32:37 +01005773 /* Control of individual bits within the mask are guarded by
5774 * the owning plane->mutex, i.e. we can never see concurrent
5775 * manipulation of individual bits. But since the bitfield as a whole
5776 * is updated using RMW, we need to use atomics in order to update
5777 * the bits.
5778 */
5779 BUILD_BUG_ON(INTEL_FRONTBUFFER_BITS_PER_PIPE * I915_MAX_PIPES >
5780 sizeof(atomic_t) * BITS_PER_BYTE);
5781
Daniel Vettera071fa02014-06-18 23:28:09 +02005782 if (old) {
Chris Wilsonfaf5bf02016-08-04 16:32:37 +01005783 WARN_ON(!(atomic_read(&old->frontbuffer_bits) & frontbuffer_bits));
5784 atomic_andnot(frontbuffer_bits, &old->frontbuffer_bits);
Daniel Vettera071fa02014-06-18 23:28:09 +02005785 }
5786
5787 if (new) {
Chris Wilsonfaf5bf02016-08-04 16:32:37 +01005788 WARN_ON(atomic_read(&new->frontbuffer_bits) & frontbuffer_bits);
5789 atomic_or(frontbuffer_bits, &new->frontbuffer_bits);
Daniel Vettera071fa02014-06-18 23:28:09 +02005790 }
5791}
5792
Dave Gordonea702992015-07-09 19:29:02 +01005793/* Allocate a new GEM object and fill it with the supplied data */
5794struct drm_i915_gem_object *
Tvrtko Ursulin12d79d72016-12-01 14:16:37 +00005795i915_gem_object_create_from_data(struct drm_i915_private *dev_priv,
Dave Gordonea702992015-07-09 19:29:02 +01005796 const void *data, size_t size)
5797{
5798 struct drm_i915_gem_object *obj;
Chris Wilsonbe062fa2017-03-17 19:46:48 +00005799 struct file *file;
5800 size_t offset;
5801 int err;
Dave Gordonea702992015-07-09 19:29:02 +01005802
Tvrtko Ursulin12d79d72016-12-01 14:16:37 +00005803 obj = i915_gem_object_create(dev_priv, round_up(size, PAGE_SIZE));
Chris Wilsonfe3db792016-04-25 13:32:13 +01005804 if (IS_ERR(obj))
Dave Gordonea702992015-07-09 19:29:02 +01005805 return obj;
5806
Christian Königc0a51fd2018-02-16 13:43:38 +01005807 GEM_BUG_ON(obj->write_domain != I915_GEM_DOMAIN_CPU);
Dave Gordonea702992015-07-09 19:29:02 +01005808
Chris Wilsonbe062fa2017-03-17 19:46:48 +00005809 file = obj->base.filp;
5810 offset = 0;
5811 do {
5812 unsigned int len = min_t(typeof(size), size, PAGE_SIZE);
5813 struct page *page;
5814 void *pgdata, *vaddr;
Dave Gordonea702992015-07-09 19:29:02 +01005815
Chris Wilsonbe062fa2017-03-17 19:46:48 +00005816 err = pagecache_write_begin(file, file->f_mapping,
5817 offset, len, 0,
5818 &page, &pgdata);
5819 if (err < 0)
5820 goto fail;
Dave Gordonea702992015-07-09 19:29:02 +01005821
Chris Wilsonbe062fa2017-03-17 19:46:48 +00005822 vaddr = kmap(page);
5823 memcpy(vaddr, data, len);
5824 kunmap(page);
5825
5826 err = pagecache_write_end(file, file->f_mapping,
5827 offset, len, len,
5828 page, pgdata);
5829 if (err < 0)
5830 goto fail;
5831
5832 size -= len;
5833 data += len;
5834 offset += len;
5835 } while (size);
Dave Gordonea702992015-07-09 19:29:02 +01005836
5837 return obj;
5838
5839fail:
Chris Wilsonf8c417c2016-07-20 13:31:53 +01005840 i915_gem_object_put(obj);
Chris Wilsonbe062fa2017-03-17 19:46:48 +00005841 return ERR_PTR(err);
Dave Gordonea702992015-07-09 19:29:02 +01005842}
Chris Wilson96d77632016-10-28 13:58:33 +01005843
5844struct scatterlist *
5845i915_gem_object_get_sg(struct drm_i915_gem_object *obj,
5846 unsigned int n,
5847 unsigned int *offset)
5848{
Chris Wilsona4f5ea62016-10-28 13:58:35 +01005849 struct i915_gem_object_page_iter *iter = &obj->mm.get_page;
Chris Wilson96d77632016-10-28 13:58:33 +01005850 struct scatterlist *sg;
5851 unsigned int idx, count;
5852
5853 might_sleep();
5854 GEM_BUG_ON(n >= obj->base.size >> PAGE_SHIFT);
Chris Wilsona4f5ea62016-10-28 13:58:35 +01005855 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
Chris Wilson96d77632016-10-28 13:58:33 +01005856
5857 /* As we iterate forward through the sg, we record each entry in a
5858 * radixtree for quick repeated (backwards) lookups. If we have seen
5859 * this index previously, we will have an entry for it.
5860 *
5861 * Initial lookup is O(N), but this is amortized to O(1) for
5862 * sequential page access (where each new request is consecutive
5863 * to the previous one). Repeated lookups are O(lg(obj->base.size)),
5864 * i.e. O(1) with a large constant!
5865 */
5866 if (n < READ_ONCE(iter->sg_idx))
5867 goto lookup;
5868
5869 mutex_lock(&iter->lock);
5870
5871 /* We prefer to reuse the last sg so that repeated lookup of this
5872 * (or the subsequent) sg are fast - comparing against the last
5873 * sg is faster than going through the radixtree.
5874 */
5875
5876 sg = iter->sg_pos;
5877 idx = iter->sg_idx;
5878 count = __sg_page_count(sg);
5879
5880 while (idx + count <= n) {
5881 unsigned long exception, i;
5882 int ret;
5883
5884 /* If we cannot allocate and insert this entry, or the
5885 * individual pages from this range, cancel updating the
5886 * sg_idx so that on this lookup we are forced to linearly
5887 * scan onwards, but on future lookups we will try the
5888 * insertion again (in which case we need to be careful of
5889 * the error return reporting that we have already inserted
5890 * this index).
5891 */
5892 ret = radix_tree_insert(&iter->radix, idx, sg);
5893 if (ret && ret != -EEXIST)
5894 goto scan;
5895
5896 exception =
5897 RADIX_TREE_EXCEPTIONAL_ENTRY |
5898 idx << RADIX_TREE_EXCEPTIONAL_SHIFT;
5899 for (i = 1; i < count; i++) {
5900 ret = radix_tree_insert(&iter->radix, idx + i,
5901 (void *)exception);
5902 if (ret && ret != -EEXIST)
5903 goto scan;
5904 }
5905
5906 idx += count;
5907 sg = ____sg_next(sg);
5908 count = __sg_page_count(sg);
5909 }
5910
5911scan:
5912 iter->sg_pos = sg;
5913 iter->sg_idx = idx;
5914
5915 mutex_unlock(&iter->lock);
5916
5917 if (unlikely(n < idx)) /* insertion completed by another thread */
5918 goto lookup;
5919
5920 /* In case we failed to insert the entry into the radixtree, we need
5921 * to look beyond the current sg.
5922 */
5923 while (idx + count <= n) {
5924 idx += count;
5925 sg = ____sg_next(sg);
5926 count = __sg_page_count(sg);
5927 }
5928
5929 *offset = n - idx;
5930 return sg;
5931
5932lookup:
5933 rcu_read_lock();
5934
5935 sg = radix_tree_lookup(&iter->radix, n);
5936 GEM_BUG_ON(!sg);
5937
5938 /* If this index is in the middle of multi-page sg entry,
5939 * the radixtree will contain an exceptional entry that points
5940 * to the start of that range. We will return the pointer to
5941 * the base page and the offset of this page within the
5942 * sg entry's range.
5943 */
5944 *offset = 0;
5945 if (unlikely(radix_tree_exception(sg))) {
5946 unsigned long base =
5947 (unsigned long)sg >> RADIX_TREE_EXCEPTIONAL_SHIFT;
5948
5949 sg = radix_tree_lookup(&iter->radix, base);
5950 GEM_BUG_ON(!sg);
5951
5952 *offset = n - base;
5953 }
5954
5955 rcu_read_unlock();
5956
5957 return sg;
5958}
5959
5960struct page *
5961i915_gem_object_get_page(struct drm_i915_gem_object *obj, unsigned int n)
5962{
5963 struct scatterlist *sg;
5964 unsigned int offset;
5965
5966 GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
5967
5968 sg = i915_gem_object_get_sg(obj, n, &offset);
5969 return nth_page(sg_page(sg), offset);
5970}
5971
5972/* Like i915_gem_object_get_page(), but mark the returned page dirty */
5973struct page *
5974i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj,
5975 unsigned int n)
5976{
5977 struct page *page;
5978
5979 page = i915_gem_object_get_page(obj, n);
Chris Wilsona4f5ea62016-10-28 13:58:35 +01005980 if (!obj->mm.dirty)
Chris Wilson96d77632016-10-28 13:58:33 +01005981 set_page_dirty(page);
5982
5983 return page;
5984}
5985
5986dma_addr_t
5987i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj,
5988 unsigned long n)
5989{
5990 struct scatterlist *sg;
5991 unsigned int offset;
5992
5993 sg = i915_gem_object_get_sg(obj, n, &offset);
5994 return sg_dma_address(sg) + (offset << PAGE_SHIFT);
5995}
Chris Wilson935a2f72017-02-13 17:15:13 +00005996
Chris Wilson8eeb7902017-07-26 19:16:01 +01005997int i915_gem_object_attach_phys(struct drm_i915_gem_object *obj, int align)
5998{
5999 struct sg_table *pages;
6000 int err;
6001
6002 if (align > obj->base.size)
6003 return -EINVAL;
6004
6005 if (obj->ops == &i915_gem_phys_ops)
6006 return 0;
6007
6008 if (obj->ops != &i915_gem_object_ops)
6009 return -EINVAL;
6010
6011 err = i915_gem_object_unbind(obj);
6012 if (err)
6013 return err;
6014
6015 mutex_lock(&obj->mm.lock);
6016
6017 if (obj->mm.madv != I915_MADV_WILLNEED) {
6018 err = -EFAULT;
6019 goto err_unlock;
6020 }
6021
6022 if (obj->mm.quirked) {
6023 err = -EFAULT;
6024 goto err_unlock;
6025 }
6026
6027 if (obj->mm.mapping) {
6028 err = -EBUSY;
6029 goto err_unlock;
6030 }
6031
Chris Wilsonf2123812017-10-16 12:40:37 +01006032 pages = fetch_and_zero(&obj->mm.pages);
6033 if (pages) {
6034 struct drm_i915_private *i915 = to_i915(obj->base.dev);
6035
6036 __i915_gem_object_reset_page_iter(obj);
6037
6038 spin_lock(&i915->mm.obj_lock);
6039 list_del(&obj->mm.link);
6040 spin_unlock(&i915->mm.obj_lock);
6041 }
6042
Chris Wilson8eeb7902017-07-26 19:16:01 +01006043 obj->ops = &i915_gem_phys_ops;
6044
Chris Wilson8fb6a5d2017-07-26 19:16:02 +01006045 err = ____i915_gem_object_get_pages(obj);
Chris Wilson8eeb7902017-07-26 19:16:01 +01006046 if (err)
6047 goto err_xfer;
6048
6049 /* Perma-pin (until release) the physical set of pages */
6050 __i915_gem_object_pin_pages(obj);
6051
6052 if (!IS_ERR_OR_NULL(pages))
6053 i915_gem_object_ops.put_pages(obj, pages);
6054 mutex_unlock(&obj->mm.lock);
6055 return 0;
6056
6057err_xfer:
6058 obj->ops = &i915_gem_object_ops;
6059 obj->mm.pages = pages;
6060err_unlock:
6061 mutex_unlock(&obj->mm.lock);
6062 return err;
6063}
6064
Chris Wilson935a2f72017-02-13 17:15:13 +00006065#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
6066#include "selftests/scatterlist.c"
Chris Wilson66d9cb52017-02-13 17:15:17 +00006067#include "selftests/mock_gem_device.c"
Chris Wilson44653982017-02-13 17:15:20 +00006068#include "selftests/huge_gem_object.c"
Matthew Auld40498662017-10-06 23:18:29 +01006069#include "selftests/huge_pages.c"
Chris Wilson8335fd62017-02-13 17:15:28 +00006070#include "selftests/i915_gem_object.c"
Chris Wilson17059452017-02-13 17:15:32 +00006071#include "selftests/i915_gem_coherency.c"
Chris Wilson935a2f72017-02-13 17:15:13 +00006072#endif