blob: ef3019c88eb79cdbeb9aa7c114ce9227238e9efa [file] [log] [blame]
Daniel Vetter76aaf222010-11-05 22:23:30 +01001/*
2 * Copyright © 2010 Daniel Vetter
Ben Widawskyc4ac5242014-02-19 22:05:47 -08003 * Copyright © 2011-2014 Intel Corporation
Daniel Vetter76aaf222010-11-05 22:23:30 +01004 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 *
24 */
25
Daniel Vetter0e46ce22014-01-08 16:10:27 +010026#include <linux/seq_file.h>
David Howells760285e2012-10-02 18:01:07 +010027#include <drm/drmP.h>
28#include <drm/i915_drm.h>
Daniel Vetter76aaf222010-11-05 22:23:30 +010029#include "i915_drv.h"
Yu Zhang5dda8fa2015-02-10 19:05:48 +080030#include "i915_vgpu.h"
Daniel Vetter76aaf222010-11-05 22:23:30 +010031#include "i915_trace.h"
32#include "intel_drv.h"
33
Tvrtko Ursulin45f8f692014-12-10 17:27:59 +000034/**
35 * DOC: Global GTT views
36 *
37 * Background and previous state
38 *
39 * Historically objects could exists (be bound) in global GTT space only as
40 * singular instances with a view representing all of the object's backing pages
41 * in a linear fashion. This view will be called a normal view.
42 *
43 * To support multiple views of the same object, where the number of mapped
44 * pages is not equal to the backing store, or where the layout of the pages
45 * is not linear, concept of a GGTT view was added.
46 *
47 * One example of an alternative view is a stereo display driven by a single
48 * image. In this case we would have a framebuffer looking like this
49 * (2x2 pages):
50 *
51 * 12
52 * 34
53 *
54 * Above would represent a normal GGTT view as normally mapped for GPU or CPU
55 * rendering. In contrast, fed to the display engine would be an alternative
56 * view which could look something like this:
57 *
58 * 1212
59 * 3434
60 *
61 * In this example both the size and layout of pages in the alternative view is
62 * different from the normal view.
63 *
64 * Implementation and usage
65 *
66 * GGTT views are implemented using VMAs and are distinguished via enum
67 * i915_ggtt_view_type and struct i915_ggtt_view.
68 *
69 * A new flavour of core GEM functions which work with GGTT bound objects were
Joonas Lahtinenec7adb62015-03-16 14:11:13 +020070 * added with the _ggtt_ infix, and sometimes with _view postfix to avoid
71 * renaming in large amounts of code. They take the struct i915_ggtt_view
72 * parameter encapsulating all metadata required to implement a view.
Tvrtko Ursulin45f8f692014-12-10 17:27:59 +000073 *
74 * As a helper for callers which are only interested in the normal view,
75 * globally const i915_ggtt_view_normal singleton instance exists. All old core
76 * GEM API functions, the ones not taking the view parameter, are operating on,
77 * or with the normal GGTT view.
78 *
79 * Code wanting to add or use a new GGTT view needs to:
80 *
81 * 1. Add a new enum with a suitable name.
82 * 2. Extend the metadata in the i915_ggtt_view structure if required.
83 * 3. Add support to i915_get_vma_pages().
84 *
85 * New views are required to build a scatter-gather table from within the
86 * i915_get_vma_pages function. This table is stored in the vma.ggtt_view and
87 * exists for the lifetime of an VMA.
88 *
89 * Core API is designed to have copy semantics which means that passed in
90 * struct i915_ggtt_view does not need to be persistent (left around after
91 * calling the core API functions).
92 *
93 */
94
Daniel Vetter70b9f6f2015-04-14 17:35:27 +020095static int
96i915_get_ggtt_vma_pages(struct i915_vma *vma);
97
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +000098const struct i915_ggtt_view i915_ggtt_view_normal;
Joonas Lahtinen9abc4642015-03-27 13:09:22 +020099const struct i915_ggtt_view i915_ggtt_view_rotated = {
100 .type = I915_GGTT_VIEW_ROTATED
101};
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +0000102
Daniel Vettercfa7c862014-04-29 11:53:58 +0200103static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
104{
Chris Wilson1893a712014-09-19 11:56:27 +0100105 bool has_aliasing_ppgtt;
106 bool has_full_ppgtt;
107
108 has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
109 has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
Chris Wilson1893a712014-09-19 11:56:27 +0100110
Yu Zhang71ba2d62015-02-10 19:05:54 +0800111 if (intel_vgpu_active(dev))
112 has_full_ppgtt = false; /* emulation is too hard */
113
Damien Lespiau70ee45e2014-11-14 15:05:59 +0000114 /*
115 * We don't allow disabling PPGTT for gen9+ as it's a requirement for
116 * execlists, the sole mechanism available to submit work.
117 */
118 if (INTEL_INFO(dev)->gen < 9 &&
119 (enable_ppgtt == 0 || !has_aliasing_ppgtt))
Daniel Vettercfa7c862014-04-29 11:53:58 +0200120 return 0;
121
122 if (enable_ppgtt == 1)
123 return 1;
124
Chris Wilson1893a712014-09-19 11:56:27 +0100125 if (enable_ppgtt == 2 && has_full_ppgtt)
Daniel Vettercfa7c862014-04-29 11:53:58 +0200126 return 2;
127
Daniel Vetter93a25a92014-03-06 09:40:43 +0100128#ifdef CONFIG_INTEL_IOMMU
129 /* Disable ppgtt on SNB if VT-d is on. */
130 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
131 DRM_INFO("Disabling PPGTT because VT-d is on\n");
Daniel Vettercfa7c862014-04-29 11:53:58 +0200132 return 0;
Daniel Vetter93a25a92014-03-06 09:40:43 +0100133 }
134#endif
135
Jesse Barnes62942ed2014-06-13 09:28:33 -0700136 /* Early VLV doesn't have this */
Ville Syrjäläca2aed6c2014-06-28 02:03:56 +0300137 if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
138 dev->pdev->revision < 0xb) {
Jesse Barnes62942ed2014-06-13 09:28:33 -0700139 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
140 return 0;
141 }
142
Michel Thierry2f82bbd2014-12-15 14:58:00 +0000143 if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists)
144 return 2;
145 else
146 return has_aliasing_ppgtt ? 1 : 0;
Daniel Vetter93a25a92014-03-06 09:40:43 +0100147}
148
Daniel Vetter70b9f6f2015-04-14 17:35:27 +0200149static int ppgtt_bind_vma(struct i915_vma *vma,
150 enum i915_cache_level cache_level,
151 u32 unused)
Daniel Vetter47552652015-04-14 17:35:24 +0200152{
153 u32 pte_flags = 0;
154
155 /* Currently applicable only to VLV */
156 if (vma->obj->gt_ro)
157 pte_flags |= PTE_READ_ONLY;
158
159 vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
160 cache_level, pte_flags);
Daniel Vetter70b9f6f2015-04-14 17:35:27 +0200161
162 return 0;
Daniel Vetter47552652015-04-14 17:35:24 +0200163}
164
165static void ppgtt_unbind_vma(struct i915_vma *vma)
166{
167 vma->vm->clear_range(vma->vm,
168 vma->node.start,
169 vma->obj->base.size,
170 true);
171}
Ben Widawsky6f65e292013-12-06 14:10:56 -0800172
Daniel Vetter2c642b02015-04-14 17:35:26 +0200173static gen8_pte_t gen8_pte_encode(dma_addr_t addr,
174 enum i915_cache_level level,
175 bool valid)
Ben Widawsky94ec8f62013-11-02 21:07:18 -0700176{
Michel Thierry07749ef2015-03-16 16:00:54 +0000177 gen8_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
Ben Widawsky94ec8f62013-11-02 21:07:18 -0700178 pte |= addr;
Ben Widawsky63c42e52014-04-18 18:04:27 -0300179
180 switch (level) {
181 case I915_CACHE_NONE:
Ben Widawskyfbe5d362013-11-04 19:56:49 -0800182 pte |= PPAT_UNCACHED_INDEX;
Ben Widawsky63c42e52014-04-18 18:04:27 -0300183 break;
184 case I915_CACHE_WT:
185 pte |= PPAT_DISPLAY_ELLC_INDEX;
186 break;
187 default:
188 pte |= PPAT_CACHED_INDEX;
189 break;
190 }
191
Ben Widawsky94ec8f62013-11-02 21:07:18 -0700192 return pte;
193}
194
Mika Kuoppalafe36f552015-06-25 18:35:16 +0300195static gen8_pde_t gen8_pde_encode(const dma_addr_t addr,
196 const enum i915_cache_level level)
Ben Widawskyb1fe6672013-11-04 21:20:14 -0800197{
Michel Thierry07749ef2015-03-16 16:00:54 +0000198 gen8_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
Ben Widawskyb1fe6672013-11-04 21:20:14 -0800199 pde |= addr;
200 if (level != I915_CACHE_NONE)
201 pde |= PPAT_CACHED_PDE_INDEX;
202 else
203 pde |= PPAT_UNCACHED_INDEX;
204 return pde;
205}
206
Michel Thierry07749ef2015-03-16 16:00:54 +0000207static gen6_pte_t snb_pte_encode(dma_addr_t addr,
208 enum i915_cache_level level,
209 bool valid, u32 unused)
Ben Widawsky54d12522012-09-24 16:44:32 -0700210{
Michel Thierry07749ef2015-03-16 16:00:54 +0000211 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Ben Widawsky54d12522012-09-24 16:44:32 -0700212 pte |= GEN6_PTE_ADDR_ENCODE(addr);
Ben Widawskye7210c32012-10-19 09:33:22 -0700213
214 switch (level) {
Chris Wilson350ec882013-08-06 13:17:02 +0100215 case I915_CACHE_L3_LLC:
216 case I915_CACHE_LLC:
217 pte |= GEN6_PTE_CACHE_LLC;
218 break;
219 case I915_CACHE_NONE:
220 pte |= GEN6_PTE_UNCACHED;
221 break;
222 default:
Daniel Vetter5f77eeb2014-12-08 16:40:10 +0100223 MISSING_CASE(level);
Chris Wilson350ec882013-08-06 13:17:02 +0100224 }
225
226 return pte;
227}
228
Michel Thierry07749ef2015-03-16 16:00:54 +0000229static gen6_pte_t ivb_pte_encode(dma_addr_t addr,
230 enum i915_cache_level level,
231 bool valid, u32 unused)
Chris Wilson350ec882013-08-06 13:17:02 +0100232{
Michel Thierry07749ef2015-03-16 16:00:54 +0000233 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Chris Wilson350ec882013-08-06 13:17:02 +0100234 pte |= GEN6_PTE_ADDR_ENCODE(addr);
235
236 switch (level) {
237 case I915_CACHE_L3_LLC:
238 pte |= GEN7_PTE_CACHE_L3_LLC;
Ben Widawskye7210c32012-10-19 09:33:22 -0700239 break;
240 case I915_CACHE_LLC:
241 pte |= GEN6_PTE_CACHE_LLC;
242 break;
243 case I915_CACHE_NONE:
Kenneth Graunke91197082013-04-22 00:53:51 -0700244 pte |= GEN6_PTE_UNCACHED;
Ben Widawskye7210c32012-10-19 09:33:22 -0700245 break;
246 default:
Daniel Vetter5f77eeb2014-12-08 16:40:10 +0100247 MISSING_CASE(level);
Ben Widawskye7210c32012-10-19 09:33:22 -0700248 }
249
Ben Widawsky54d12522012-09-24 16:44:32 -0700250 return pte;
251}
252
Michel Thierry07749ef2015-03-16 16:00:54 +0000253static gen6_pte_t byt_pte_encode(dma_addr_t addr,
254 enum i915_cache_level level,
255 bool valid, u32 flags)
Kenneth Graunke93c34e72013-04-22 00:53:50 -0700256{
Michel Thierry07749ef2015-03-16 16:00:54 +0000257 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Kenneth Graunke93c34e72013-04-22 00:53:50 -0700258 pte |= GEN6_PTE_ADDR_ENCODE(addr);
259
Akash Goel24f3a8c2014-06-17 10:59:42 +0530260 if (!(flags & PTE_READ_ONLY))
261 pte |= BYT_PTE_WRITEABLE;
Kenneth Graunke93c34e72013-04-22 00:53:50 -0700262
263 if (level != I915_CACHE_NONE)
264 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
265
266 return pte;
267}
268
Michel Thierry07749ef2015-03-16 16:00:54 +0000269static gen6_pte_t hsw_pte_encode(dma_addr_t addr,
270 enum i915_cache_level level,
271 bool valid, u32 unused)
Kenneth Graunke91197082013-04-22 00:53:51 -0700272{
Michel Thierry07749ef2015-03-16 16:00:54 +0000273 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Ben Widawsky0d8ff152013-07-04 11:02:03 -0700274 pte |= HSW_PTE_ADDR_ENCODE(addr);
Kenneth Graunke91197082013-04-22 00:53:51 -0700275
276 if (level != I915_CACHE_NONE)
Ben Widawsky87a6b682013-08-04 23:47:29 -0700277 pte |= HSW_WB_LLC_AGE3;
Kenneth Graunke91197082013-04-22 00:53:51 -0700278
279 return pte;
280}
281
Michel Thierry07749ef2015-03-16 16:00:54 +0000282static gen6_pte_t iris_pte_encode(dma_addr_t addr,
283 enum i915_cache_level level,
284 bool valid, u32 unused)
Ben Widawsky4d15c142013-07-04 11:02:06 -0700285{
Michel Thierry07749ef2015-03-16 16:00:54 +0000286 gen6_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Ben Widawsky4d15c142013-07-04 11:02:06 -0700287 pte |= HSW_PTE_ADDR_ENCODE(addr);
288
Chris Wilson651d7942013-08-08 14:41:10 +0100289 switch (level) {
290 case I915_CACHE_NONE:
291 break;
292 case I915_CACHE_WT:
Chris Wilsonc51e9702013-11-22 10:37:53 +0000293 pte |= HSW_WT_ELLC_LLC_AGE3;
Chris Wilson651d7942013-08-08 14:41:10 +0100294 break;
295 default:
Chris Wilsonc51e9702013-11-22 10:37:53 +0000296 pte |= HSW_WB_ELLC_LLC_AGE3;
Chris Wilson651d7942013-08-08 14:41:10 +0100297 break;
298 }
Ben Widawsky4d15c142013-07-04 11:02:06 -0700299
300 return pte;
301}
302
Mika Kuoppalac114f762015-06-25 18:35:13 +0300303static int __setup_page_dma(struct drm_device *dev,
304 struct i915_page_dma *p, gfp_t flags)
Ben Widawsky678d96f2015-03-16 16:00:56 +0000305{
306 struct device *device = &dev->pdev->dev;
307
Mika Kuoppalac114f762015-06-25 18:35:13 +0300308 p->page = alloc_page(flags);
Mika Kuoppala44159dd2015-06-25 18:35:07 +0300309 if (!p->page)
Michel Thierry1266cdb2015-03-24 17:06:33 +0000310 return -ENOMEM;
311
Mika Kuoppala44159dd2015-06-25 18:35:07 +0300312 p->daddr = dma_map_page(device,
313 p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL);
314
315 if (dma_mapping_error(device, p->daddr)) {
316 __free_page(p->page);
317 return -EINVAL;
318 }
319
Michel Thierry1266cdb2015-03-24 17:06:33 +0000320 return 0;
Ben Widawsky678d96f2015-03-16 16:00:56 +0000321}
322
Mika Kuoppalac114f762015-06-25 18:35:13 +0300323static int setup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
324{
325 return __setup_page_dma(dev, p, GFP_KERNEL);
326}
327
Mika Kuoppala44159dd2015-06-25 18:35:07 +0300328static void cleanup_page_dma(struct drm_device *dev, struct i915_page_dma *p)
329{
330 if (WARN_ON(!p->page))
331 return;
332
333 dma_unmap_page(&dev->pdev->dev, p->daddr, 4096, PCI_DMA_BIDIRECTIONAL);
334 __free_page(p->page);
335 memset(p, 0, sizeof(*p));
336}
337
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +0300338static void *kmap_page_dma(struct i915_page_dma *p)
Mika Kuoppala73eeea52015-06-25 18:35:10 +0300339{
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +0300340 return kmap_atomic(p->page);
341}
Mika Kuoppala73eeea52015-06-25 18:35:10 +0300342
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +0300343/* We use the flushing unmap only with ppgtt structures:
344 * page directories, page tables and scratch pages.
345 */
346static void kunmap_page_dma(struct drm_device *dev, void *vaddr)
347{
Mika Kuoppala73eeea52015-06-25 18:35:10 +0300348 /* There are only few exceptions for gen >=6. chv and bxt.
349 * And we are not sure about the latter so play safe for now.
350 */
351 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
352 drm_clflush_virt_range(vaddr, PAGE_SIZE);
353
354 kunmap_atomic(vaddr);
355}
356
Mika Kuoppala567047b2015-06-25 18:35:12 +0300357#define kmap_px(px) kmap_page_dma(px_base(px))
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +0300358#define kunmap_px(ppgtt, vaddr) kunmap_page_dma((ppgtt)->base.dev, (vaddr))
359
Mika Kuoppala567047b2015-06-25 18:35:12 +0300360#define setup_px(dev, px) setup_page_dma((dev), px_base(px))
361#define cleanup_px(dev, px) cleanup_page_dma((dev), px_base(px))
362#define fill_px(dev, px, v) fill_page_dma((dev), px_base(px), (v))
363#define fill32_px(dev, px, v) fill_page_dma_32((dev), px_base(px), (v))
364
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +0300365static void fill_page_dma(struct drm_device *dev, struct i915_page_dma *p,
366 const uint64_t val)
367{
368 int i;
369 uint64_t * const vaddr = kmap_page_dma(p);
370
371 for (i = 0; i < 512; i++)
372 vaddr[i] = val;
373
374 kunmap_page_dma(dev, vaddr);
375}
376
Mika Kuoppala73eeea52015-06-25 18:35:10 +0300377static void fill_page_dma_32(struct drm_device *dev, struct i915_page_dma *p,
378 const uint32_t val32)
379{
380 uint64_t v = val32;
381
382 v = v << 32 | val32;
383
384 fill_page_dma(dev, p, v);
385}
386
Mika Kuoppala8a1ebd72015-05-22 20:04:59 +0300387static struct i915_page_table *alloc_pt(struct drm_device *dev)
Ben Widawsky06fda602015-02-24 16:22:36 +0000388{
Michel Thierryec565b32015-04-08 12:13:23 +0100389 struct i915_page_table *pt;
Ben Widawsky678d96f2015-03-16 16:00:56 +0000390 const size_t count = INTEL_INFO(dev)->gen >= 8 ?
391 GEN8_PTES : GEN6_PTES;
392 int ret = -ENOMEM;
Ben Widawsky06fda602015-02-24 16:22:36 +0000393
394 pt = kzalloc(sizeof(*pt), GFP_KERNEL);
395 if (!pt)
396 return ERR_PTR(-ENOMEM);
397
Ben Widawsky678d96f2015-03-16 16:00:56 +0000398 pt->used_ptes = kcalloc(BITS_TO_LONGS(count), sizeof(*pt->used_ptes),
399 GFP_KERNEL);
400
401 if (!pt->used_ptes)
402 goto fail_bitmap;
403
Mika Kuoppala567047b2015-06-25 18:35:12 +0300404 ret = setup_px(dev, pt);
Ben Widawsky678d96f2015-03-16 16:00:56 +0000405 if (ret)
Mika Kuoppala44159dd2015-06-25 18:35:07 +0300406 goto fail_page_m;
Ben Widawsky06fda602015-02-24 16:22:36 +0000407
408 return pt;
Ben Widawsky678d96f2015-03-16 16:00:56 +0000409
Mika Kuoppala44159dd2015-06-25 18:35:07 +0300410fail_page_m:
Ben Widawsky678d96f2015-03-16 16:00:56 +0000411 kfree(pt->used_ptes);
412fail_bitmap:
413 kfree(pt);
414
415 return ERR_PTR(ret);
Ben Widawsky06fda602015-02-24 16:22:36 +0000416}
417
Mika Kuoppala2e906be2015-06-30 18:16:37 +0300418static void free_pt(struct drm_device *dev, struct i915_page_table *pt)
Ben Widawsky06fda602015-02-24 16:22:36 +0000419{
Mika Kuoppala2e906be2015-06-30 18:16:37 +0300420 cleanup_px(dev, pt);
421 kfree(pt->used_ptes);
422 kfree(pt);
423}
424
425static void gen8_initialize_pt(struct i915_address_space *vm,
426 struct i915_page_table *pt)
427{
428 gen8_pte_t scratch_pte;
429
430 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
431 I915_CACHE_LLC, true);
432
433 fill_px(vm->dev, pt, scratch_pte);
434}
435
436static void gen6_initialize_pt(struct i915_address_space *vm,
437 struct i915_page_table *pt)
438{
439 gen6_pte_t scratch_pte;
440
441 WARN_ON(px_dma(vm->scratch_page) == 0);
442
443 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
444 I915_CACHE_LLC, true, 0);
445
446 fill32_px(vm->dev, pt, scratch_pte);
Ben Widawsky06fda602015-02-24 16:22:36 +0000447}
448
Mika Kuoppala8a1ebd72015-05-22 20:04:59 +0300449static struct i915_page_directory *alloc_pd(struct drm_device *dev)
Ben Widawsky06fda602015-02-24 16:22:36 +0000450{
Michel Thierryec565b32015-04-08 12:13:23 +0100451 struct i915_page_directory *pd;
Michel Thierry33c88192015-04-08 12:13:33 +0100452 int ret = -ENOMEM;
Ben Widawsky06fda602015-02-24 16:22:36 +0000453
454 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
455 if (!pd)
456 return ERR_PTR(-ENOMEM);
457
Michel Thierry33c88192015-04-08 12:13:33 +0100458 pd->used_pdes = kcalloc(BITS_TO_LONGS(I915_PDES),
459 sizeof(*pd->used_pdes), GFP_KERNEL);
460 if (!pd->used_pdes)
Mika Kuoppalaa08e1112015-06-25 18:35:08 +0300461 goto fail_bitmap;
Michel Thierry33c88192015-04-08 12:13:33 +0100462
Mika Kuoppala567047b2015-06-25 18:35:12 +0300463 ret = setup_px(dev, pd);
Michel Thierry33c88192015-04-08 12:13:33 +0100464 if (ret)
Mika Kuoppalaa08e1112015-06-25 18:35:08 +0300465 goto fail_page_m;
Michel Thierrye5815a22015-04-08 12:13:32 +0100466
Ben Widawsky06fda602015-02-24 16:22:36 +0000467 return pd;
Michel Thierry33c88192015-04-08 12:13:33 +0100468
Mika Kuoppalaa08e1112015-06-25 18:35:08 +0300469fail_page_m:
Michel Thierry33c88192015-04-08 12:13:33 +0100470 kfree(pd->used_pdes);
Mika Kuoppalaa08e1112015-06-25 18:35:08 +0300471fail_bitmap:
Michel Thierry33c88192015-04-08 12:13:33 +0100472 kfree(pd);
473
474 return ERR_PTR(ret);
Ben Widawsky06fda602015-02-24 16:22:36 +0000475}
476
Mika Kuoppala2e906be2015-06-30 18:16:37 +0300477static void free_pd(struct drm_device *dev, struct i915_page_directory *pd)
478{
479 if (px_page(pd)) {
480 cleanup_px(dev, pd);
481 kfree(pd->used_pdes);
482 kfree(pd);
483 }
484}
485
486static void gen8_initialize_pd(struct i915_address_space *vm,
487 struct i915_page_directory *pd)
488{
489 gen8_pde_t scratch_pde;
490
491 scratch_pde = gen8_pde_encode(px_dma(vm->scratch_pt), I915_CACHE_LLC);
492
493 fill_px(vm->dev, pd, scratch_pde);
494}
495
496static int alloc_scratch_page(struct i915_address_space *vm)
497{
498 struct i915_page_scratch *sp;
499 int ret;
500
501 WARN_ON(vm->scratch_page);
502
503 sp = kzalloc(sizeof(*sp), GFP_KERNEL);
504 if (sp == NULL)
505 return -ENOMEM;
506
507 ret = __setup_page_dma(vm->dev, px_base(sp), GFP_DMA32 | __GFP_ZERO);
508 if (ret) {
509 kfree(sp);
510 return ret;
511 }
512
513 set_pages_uc(px_page(sp), 1);
514
515 vm->scratch_page = sp;
516
517 return 0;
518}
519
520static void free_scratch_page(struct i915_address_space *vm)
521{
522 struct i915_page_scratch *sp = vm->scratch_page;
523
524 set_pages_wb(px_page(sp), 1);
525
526 cleanup_px(vm->dev, sp);
527 kfree(sp);
528
529 vm->scratch_page = NULL;
530}
531
Ben Widawsky94e409c2013-11-04 22:29:36 -0800532/* Broadwell Page Directory Pointer Descriptors */
John Harrisone85b26d2015-05-29 17:43:56 +0100533static int gen8_write_pdp(struct drm_i915_gem_request *req,
Michel Thierry7cb6d7a2015-04-08 12:13:29 +0100534 unsigned entry,
535 dma_addr_t addr)
Ben Widawsky94e409c2013-11-04 22:29:36 -0800536{
John Harrisone85b26d2015-05-29 17:43:56 +0100537 struct intel_engine_cs *ring = req->ring;
Ben Widawsky94e409c2013-11-04 22:29:36 -0800538 int ret;
539
540 BUG_ON(entry >= 4);
541
John Harrison5fb9de12015-05-29 17:44:07 +0100542 ret = intel_ring_begin(req, 6);
Ben Widawsky94e409c2013-11-04 22:29:36 -0800543 if (ret)
544 return ret;
545
546 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
547 intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
Michel Thierry7cb6d7a2015-04-08 12:13:29 +0100548 intel_ring_emit(ring, upper_32_bits(addr));
Ben Widawsky94e409c2013-11-04 22:29:36 -0800549 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
550 intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
Michel Thierry7cb6d7a2015-04-08 12:13:29 +0100551 intel_ring_emit(ring, lower_32_bits(addr));
Ben Widawsky94e409c2013-11-04 22:29:36 -0800552 intel_ring_advance(ring);
553
554 return 0;
555}
556
Ben Widawskyeeb94882013-12-06 14:11:10 -0800557static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
John Harrisone85b26d2015-05-29 17:43:56 +0100558 struct drm_i915_gem_request *req)
Ben Widawsky94e409c2013-11-04 22:29:36 -0800559{
Ben Widawskyeeb94882013-12-06 14:11:10 -0800560 int i, ret;
Ben Widawsky94e409c2013-11-04 22:29:36 -0800561
Michel Thierry7cb6d7a2015-04-08 12:13:29 +0100562 for (i = GEN8_LEGACY_PDPES - 1; i >= 0; i--) {
Mika Kuoppalad852c7b2015-06-25 18:35:06 +0300563 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i);
564
John Harrisone85b26d2015-05-29 17:43:56 +0100565 ret = gen8_write_pdp(req, i, pd_daddr);
Ben Widawskyeeb94882013-12-06 14:11:10 -0800566 if (ret)
567 return ret;
Ben Widawsky94e409c2013-11-04 22:29:36 -0800568 }
Ben Widawskyd595bd42013-11-25 09:54:32 -0800569
Ben Widawskyeeb94882013-12-06 14:11:10 -0800570 return 0;
Ben Widawsky94e409c2013-11-04 22:29:36 -0800571}
572
Ben Widawsky459108b2013-11-02 21:07:23 -0700573static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
Ben Widawsky782f1492014-02-20 11:50:33 -0800574 uint64_t start,
575 uint64_t length,
Ben Widawsky459108b2013-11-02 21:07:23 -0700576 bool use_scratch)
577{
578 struct i915_hw_ppgtt *ppgtt =
579 container_of(vm, struct i915_hw_ppgtt, base);
Michel Thierry07749ef2015-03-16 16:00:54 +0000580 gen8_pte_t *pt_vaddr, scratch_pte;
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800581 unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
582 unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
583 unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
Ben Widawsky782f1492014-02-20 11:50:33 -0800584 unsigned num_entries = length >> PAGE_SHIFT;
Ben Widawsky459108b2013-11-02 21:07:23 -0700585 unsigned last_pte, i;
586
Mika Kuoppalac114f762015-06-25 18:35:13 +0300587 scratch_pte = gen8_pte_encode(px_dma(ppgtt->base.scratch_page),
Ben Widawsky459108b2013-11-02 21:07:23 -0700588 I915_CACHE_LLC, use_scratch);
589
590 while (num_entries) {
Michel Thierryec565b32015-04-08 12:13:23 +0100591 struct i915_page_directory *pd;
592 struct i915_page_table *pt;
Ben Widawsky06fda602015-02-24 16:22:36 +0000593
594 if (WARN_ON(!ppgtt->pdp.page_directory[pdpe]))
595 continue;
596
597 pd = ppgtt->pdp.page_directory[pdpe];
598
599 if (WARN_ON(!pd->page_table[pde]))
600 continue;
601
602 pt = pd->page_table[pde];
603
Mika Kuoppala567047b2015-06-25 18:35:12 +0300604 if (WARN_ON(!px_page(pt)))
Ben Widawsky06fda602015-02-24 16:22:36 +0000605 continue;
606
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800607 last_pte = pte + num_entries;
Michel Thierry07749ef2015-03-16 16:00:54 +0000608 if (last_pte > GEN8_PTES)
609 last_pte = GEN8_PTES;
Ben Widawsky459108b2013-11-02 21:07:23 -0700610
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +0300611 pt_vaddr = kmap_px(pt);
Ben Widawsky459108b2013-11-02 21:07:23 -0700612
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800613 for (i = pte; i < last_pte; i++) {
Ben Widawsky459108b2013-11-02 21:07:23 -0700614 pt_vaddr[i] = scratch_pte;
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800615 num_entries--;
616 }
Ben Widawsky459108b2013-11-02 21:07:23 -0700617
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +0300618 kunmap_px(ppgtt, pt);
Ben Widawsky459108b2013-11-02 21:07:23 -0700619
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800620 pte = 0;
Michel Thierry07749ef2015-03-16 16:00:54 +0000621 if (++pde == I915_PDES) {
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800622 pdpe++;
623 pde = 0;
624 }
Ben Widawsky459108b2013-11-02 21:07:23 -0700625 }
626}
627
Ben Widawsky9df15b42013-11-02 21:07:24 -0700628static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
629 struct sg_table *pages,
Ben Widawsky782f1492014-02-20 11:50:33 -0800630 uint64_t start,
Akash Goel24f3a8c2014-06-17 10:59:42 +0530631 enum i915_cache_level cache_level, u32 unused)
Ben Widawsky9df15b42013-11-02 21:07:24 -0700632{
633 struct i915_hw_ppgtt *ppgtt =
634 container_of(vm, struct i915_hw_ppgtt, base);
Michel Thierry07749ef2015-03-16 16:00:54 +0000635 gen8_pte_t *pt_vaddr;
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800636 unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
637 unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
638 unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
Ben Widawsky9df15b42013-11-02 21:07:24 -0700639 struct sg_page_iter sg_iter;
640
Chris Wilson6f1cc992013-12-31 15:50:31 +0000641 pt_vaddr = NULL;
Ben Widawsky9df15b42013-11-02 21:07:24 -0700642
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800643 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
Ben Widawsky76643602015-01-22 17:01:24 +0000644 if (WARN_ON(pdpe >= GEN8_LEGACY_PDPES))
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800645 break;
646
Ben Widawskyd7b3de92015-02-24 16:22:34 +0000647 if (pt_vaddr == NULL) {
Michel Thierryec565b32015-04-08 12:13:23 +0100648 struct i915_page_directory *pd = ppgtt->pdp.page_directory[pdpe];
649 struct i915_page_table *pt = pd->page_table[pde];
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +0300650 pt_vaddr = kmap_px(pt);
Ben Widawskyd7b3de92015-02-24 16:22:34 +0000651 }
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800652
653 pt_vaddr[pte] =
Chris Wilson6f1cc992013-12-31 15:50:31 +0000654 gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
655 cache_level, true);
Michel Thierry07749ef2015-03-16 16:00:54 +0000656 if (++pte == GEN8_PTES) {
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +0300657 kunmap_px(ppgtt, pt_vaddr);
Chris Wilson6f1cc992013-12-31 15:50:31 +0000658 pt_vaddr = NULL;
Michel Thierry07749ef2015-03-16 16:00:54 +0000659 if (++pde == I915_PDES) {
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800660 pdpe++;
661 pde = 0;
662 }
663 pte = 0;
Ben Widawsky9df15b42013-11-02 21:07:24 -0700664 }
665 }
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +0300666
667 if (pt_vaddr)
668 kunmap_px(ppgtt, pt_vaddr);
Ben Widawsky9df15b42013-11-02 21:07:24 -0700669}
670
Michel Thierryf37c0502015-06-10 17:46:39 +0100671static void gen8_free_page_tables(struct drm_device *dev,
672 struct i915_page_directory *pd)
Ben Widawskyb45a6712014-02-12 14:28:44 -0800673{
674 int i;
675
Mika Kuoppala567047b2015-06-25 18:35:12 +0300676 if (!px_page(pd))
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800677 return;
Ben Widawskyb45a6712014-02-12 14:28:44 -0800678
Michel Thierry33c88192015-04-08 12:13:33 +0100679 for_each_set_bit(i, pd->used_pdes, I915_PDES) {
Ben Widawsky06fda602015-02-24 16:22:36 +0000680 if (WARN_ON(!pd->page_table[i]))
681 continue;
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800682
Mika Kuoppalaa08e1112015-06-25 18:35:08 +0300683 free_pt(dev, pd->page_table[i]);
Ben Widawsky06fda602015-02-24 16:22:36 +0000684 pd->page_table[i] = NULL;
685 }
Ben Widawskyd7b3de92015-02-24 16:22:34 +0000686}
687
Daniel Vetter061dd492015-04-14 17:35:13 +0200688static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800689{
Daniel Vetter061dd492015-04-14 17:35:13 +0200690 struct i915_hw_ppgtt *ppgtt =
691 container_of(vm, struct i915_hw_ppgtt, base);
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800692 int i;
693
Michel Thierry33c88192015-04-08 12:13:33 +0100694 for_each_set_bit(i, ppgtt->pdp.used_pdpes, GEN8_LEGACY_PDPES) {
Ben Widawsky06fda602015-02-24 16:22:36 +0000695 if (WARN_ON(!ppgtt->pdp.page_directory[i]))
696 continue;
697
Michel Thierryf37c0502015-06-10 17:46:39 +0100698 gen8_free_page_tables(ppgtt->base.dev,
699 ppgtt->pdp.page_directory[i]);
Mika Kuoppalaa08e1112015-06-25 18:35:08 +0300700 free_pd(ppgtt->base.dev, ppgtt->pdp.page_directory[i]);
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800701 }
Michel Thierry69876be2015-04-08 12:13:27 +0100702
Mika Kuoppala79ab9372015-06-25 18:35:17 +0300703 free_pd(vm->dev, vm->scratch_pd);
704 free_pt(vm->dev, vm->scratch_pt);
Ben Widawskyb45a6712014-02-12 14:28:44 -0800705}
706
Michel Thierryd7b26332015-04-08 12:13:34 +0100707/**
708 * gen8_ppgtt_alloc_pagetabs() - Allocate page tables for VA range.
709 * @ppgtt: Master ppgtt structure.
710 * @pd: Page directory for this address range.
711 * @start: Starting virtual address to begin allocations.
712 * @length Size of the allocations.
713 * @new_pts: Bitmap set by function with new allocations. Likely used by the
714 * caller to free on error.
715 *
716 * Allocate the required number of page tables. Extremely similar to
717 * gen8_ppgtt_alloc_page_directories(). The main difference is here we are limited by
718 * the page directory boundary (instead of the page directory pointer). That
719 * boundary is 1GB virtual. Therefore, unlike gen8_ppgtt_alloc_page_directories(), it is
720 * possible, and likely that the caller will need to use multiple calls of this
721 * function to achieve the appropriate allocation.
722 *
723 * Return: 0 if success; negative error code otherwise.
724 */
Michel Thierrye5815a22015-04-08 12:13:32 +0100725static int gen8_ppgtt_alloc_pagetabs(struct i915_hw_ppgtt *ppgtt,
726 struct i915_page_directory *pd,
Michel Thierry5441f0c2015-04-08 12:13:28 +0100727 uint64_t start,
Michel Thierryd7b26332015-04-08 12:13:34 +0100728 uint64_t length,
729 unsigned long *new_pts)
Ben Widawskyd7b3de92015-02-24 16:22:34 +0000730{
Michel Thierrye5815a22015-04-08 12:13:32 +0100731 struct drm_device *dev = ppgtt->base.dev;
Michel Thierryd7b26332015-04-08 12:13:34 +0100732 struct i915_page_table *pt;
Michel Thierry5441f0c2015-04-08 12:13:28 +0100733 uint64_t temp;
734 uint32_t pde;
Ben Widawskyd7b3de92015-02-24 16:22:34 +0000735
Michel Thierryd7b26332015-04-08 12:13:34 +0100736 gen8_for_each_pde(pt, pd, start, length, temp, pde) {
737 /* Don't reallocate page tables */
738 if (pt) {
739 /* Scratch is never allocated this way */
Mika Kuoppala79ab9372015-06-25 18:35:17 +0300740 WARN_ON(pt == ppgtt->base.scratch_pt);
Michel Thierryd7b26332015-04-08 12:13:34 +0100741 continue;
742 }
743
Mika Kuoppala8a1ebd72015-05-22 20:04:59 +0300744 pt = alloc_pt(dev);
Michel Thierryd7b26332015-04-08 12:13:34 +0100745 if (IS_ERR(pt))
Ben Widawsky06fda602015-02-24 16:22:36 +0000746 goto unwind_out;
Michel Thierry5441f0c2015-04-08 12:13:28 +0100747
Michel Thierryd7b26332015-04-08 12:13:34 +0100748 gen8_initialize_pt(&ppgtt->base, pt);
749 pd->page_table[pde] = pt;
Mika Kuoppala966082c2015-06-25 18:35:19 +0300750 __set_bit(pde, new_pts);
Ben Widawskyd7b3de92015-02-24 16:22:34 +0000751 }
752
753 return 0;
754
755unwind_out:
Michel Thierryd7b26332015-04-08 12:13:34 +0100756 for_each_set_bit(pde, new_pts, I915_PDES)
Mika Kuoppalaa08e1112015-06-25 18:35:08 +0300757 free_pt(dev, pd->page_table[pde]);
Ben Widawskyd7b3de92015-02-24 16:22:34 +0000758
759 return -ENOMEM;
760}
761
Michel Thierryd7b26332015-04-08 12:13:34 +0100762/**
763 * gen8_ppgtt_alloc_page_directories() - Allocate page directories for VA range.
764 * @ppgtt: Master ppgtt structure.
765 * @pdp: Page directory pointer for this address range.
766 * @start: Starting virtual address to begin allocations.
767 * @length Size of the allocations.
768 * @new_pds Bitmap set by function with new allocations. Likely used by the
769 * caller to free on error.
770 *
771 * Allocate the required number of page directories starting at the pde index of
772 * @start, and ending at the pde index @start + @length. This function will skip
773 * over already allocated page directories within the range, and only allocate
774 * new ones, setting the appropriate pointer within the pdp as well as the
775 * correct position in the bitmap @new_pds.
776 *
777 * The function will only allocate the pages within the range for a give page
778 * directory pointer. In other words, if @start + @length straddles a virtually
779 * addressed PDP boundary (512GB for 4k pages), there will be more allocations
780 * required by the caller, This is not currently possible, and the BUG in the
781 * code will prevent it.
782 *
783 * Return: 0 if success; negative error code otherwise.
784 */
Michel Thierryc488dbb2015-04-08 12:13:31 +0100785static int gen8_ppgtt_alloc_page_directories(struct i915_hw_ppgtt *ppgtt,
786 struct i915_page_directory_pointer *pdp,
Michel Thierry69876be2015-04-08 12:13:27 +0100787 uint64_t start,
Michel Thierryd7b26332015-04-08 12:13:34 +0100788 uint64_t length,
789 unsigned long *new_pds)
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800790{
Michel Thierrye5815a22015-04-08 12:13:32 +0100791 struct drm_device *dev = ppgtt->base.dev;
Michel Thierryd7b26332015-04-08 12:13:34 +0100792 struct i915_page_directory *pd;
Michel Thierry69876be2015-04-08 12:13:27 +0100793 uint64_t temp;
794 uint32_t pdpe;
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800795
Michel Thierryd7b26332015-04-08 12:13:34 +0100796 WARN_ON(!bitmap_empty(new_pds, GEN8_LEGACY_PDPES));
797
Michel Thierryd7b26332015-04-08 12:13:34 +0100798 gen8_for_each_pdpe(pd, pdp, start, length, temp, pdpe) {
799 if (pd)
800 continue;
Michel Thierry33c88192015-04-08 12:13:33 +0100801
Mika Kuoppala8a1ebd72015-05-22 20:04:59 +0300802 pd = alloc_pd(dev);
Michel Thierryd7b26332015-04-08 12:13:34 +0100803 if (IS_ERR(pd))
Ben Widawskyd7b3de92015-02-24 16:22:34 +0000804 goto unwind_out;
Michel Thierry69876be2015-04-08 12:13:27 +0100805
Michel Thierryd7b26332015-04-08 12:13:34 +0100806 gen8_initialize_pd(&ppgtt->base, pd);
807 pdp->page_directory[pdpe] = pd;
Mika Kuoppala966082c2015-06-25 18:35:19 +0300808 __set_bit(pdpe, new_pds);
Ben Widawskyd7b3de92015-02-24 16:22:34 +0000809 }
810
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800811 return 0;
Ben Widawskyd7b3de92015-02-24 16:22:34 +0000812
813unwind_out:
Michel Thierryd7b26332015-04-08 12:13:34 +0100814 for_each_set_bit(pdpe, new_pds, GEN8_LEGACY_PDPES)
Mika Kuoppalaa08e1112015-06-25 18:35:08 +0300815 free_pd(dev, pdp->page_directory[pdpe]);
Ben Widawskyd7b3de92015-02-24 16:22:34 +0000816
817 return -ENOMEM;
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800818}
819
Michel Thierryd7b26332015-04-08 12:13:34 +0100820static void
821free_gen8_temp_bitmaps(unsigned long *new_pds, unsigned long **new_pts)
822{
823 int i;
824
825 for (i = 0; i < GEN8_LEGACY_PDPES; i++)
826 kfree(new_pts[i]);
827 kfree(new_pts);
828 kfree(new_pds);
829}
830
831/* Fills in the page directory bitmap, and the array of page tables bitmap. Both
832 * of these are based on the number of PDPEs in the system.
833 */
834static
835int __must_check alloc_gen8_temp_bitmaps(unsigned long **new_pds,
836 unsigned long ***new_pts)
837{
838 int i;
839 unsigned long *pds;
840 unsigned long **pts;
841
842 pds = kcalloc(BITS_TO_LONGS(GEN8_LEGACY_PDPES), sizeof(unsigned long), GFP_KERNEL);
843 if (!pds)
844 return -ENOMEM;
845
846 pts = kcalloc(GEN8_LEGACY_PDPES, sizeof(unsigned long *), GFP_KERNEL);
847 if (!pts) {
848 kfree(pds);
849 return -ENOMEM;
850 }
851
852 for (i = 0; i < GEN8_LEGACY_PDPES; i++) {
853 pts[i] = kcalloc(BITS_TO_LONGS(I915_PDES),
854 sizeof(unsigned long), GFP_KERNEL);
855 if (!pts[i])
856 goto err_out;
857 }
858
859 *new_pds = pds;
860 *new_pts = pts;
861
862 return 0;
863
864err_out:
865 free_gen8_temp_bitmaps(pds, pts);
866 return -ENOMEM;
867}
868
Mika Kuoppala5b7e4c9c2015-06-25 18:35:03 +0300869/* PDE TLBs are a pain to invalidate on GEN8+. When we modify
870 * the page table structures, we mark them dirty so that
871 * context switching/execlist queuing code takes extra steps
872 * to ensure that tlbs are flushed.
873 */
874static void mark_tlbs_dirty(struct i915_hw_ppgtt *ppgtt)
875{
876 ppgtt->pd_dirty_rings = INTEL_INFO(ppgtt->base.dev)->ring_mask;
877}
878
Michel Thierrye5815a22015-04-08 12:13:32 +0100879static int gen8_alloc_va_range(struct i915_address_space *vm,
880 uint64_t start,
881 uint64_t length)
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800882{
Michel Thierrye5815a22015-04-08 12:13:32 +0100883 struct i915_hw_ppgtt *ppgtt =
884 container_of(vm, struct i915_hw_ppgtt, base);
Michel Thierryd7b26332015-04-08 12:13:34 +0100885 unsigned long *new_page_dirs, **new_page_tables;
Michel Thierry5441f0c2015-04-08 12:13:28 +0100886 struct i915_page_directory *pd;
Michel Thierry33c88192015-04-08 12:13:33 +0100887 const uint64_t orig_start = start;
888 const uint64_t orig_length = length;
Michel Thierry5441f0c2015-04-08 12:13:28 +0100889 uint64_t temp;
890 uint32_t pdpe;
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800891 int ret;
892
Michel Thierryd7b26332015-04-08 12:13:34 +0100893 /* Wrap is never okay since we can only represent 48b, and we don't
894 * actually use the other side of the canonical address space.
895 */
896 if (WARN_ON(start + length < start))
Mika Kuoppalaa05d80e2015-06-25 18:35:04 +0300897 return -ENODEV;
898
899 if (WARN_ON(start + length > ppgtt->base.total))
900 return -ENODEV;
Michel Thierryd7b26332015-04-08 12:13:34 +0100901
902 ret = alloc_gen8_temp_bitmaps(&new_page_dirs, &new_page_tables);
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800903 if (ret)
904 return ret;
905
Michel Thierryd7b26332015-04-08 12:13:34 +0100906 /* Do the allocations first so we can easily bail out */
907 ret = gen8_ppgtt_alloc_page_directories(ppgtt, &ppgtt->pdp, start, length,
908 new_page_dirs);
909 if (ret) {
910 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
911 return ret;
912 }
913
914 /* For every page directory referenced, allocate page tables */
Michel Thierry5441f0c2015-04-08 12:13:28 +0100915 gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) {
Michel Thierryd7b26332015-04-08 12:13:34 +0100916 ret = gen8_ppgtt_alloc_pagetabs(ppgtt, pd, start, length,
917 new_page_tables[pdpe]);
Michel Thierry5441f0c2015-04-08 12:13:28 +0100918 if (ret)
919 goto err_out;
Michel Thierry5441f0c2015-04-08 12:13:28 +0100920 }
921
Michel Thierry33c88192015-04-08 12:13:33 +0100922 start = orig_start;
923 length = orig_length;
924
Michel Thierryd7b26332015-04-08 12:13:34 +0100925 /* Allocations have completed successfully, so set the bitmaps, and do
926 * the mappings. */
Michel Thierry33c88192015-04-08 12:13:33 +0100927 gen8_for_each_pdpe(pd, &ppgtt->pdp, start, length, temp, pdpe) {
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +0300928 gen8_pde_t *const page_directory = kmap_px(pd);
Michel Thierry33c88192015-04-08 12:13:33 +0100929 struct i915_page_table *pt;
930 uint64_t pd_len = gen8_clamp_pd(start, length);
931 uint64_t pd_start = start;
932 uint32_t pde;
933
Michel Thierryd7b26332015-04-08 12:13:34 +0100934 /* Every pd should be allocated, we just did that above. */
935 WARN_ON(!pd);
936
937 gen8_for_each_pde(pt, pd, pd_start, pd_len, temp, pde) {
938 /* Same reasoning as pd */
939 WARN_ON(!pt);
940 WARN_ON(!pd_len);
941 WARN_ON(!gen8_pte_count(pd_start, pd_len));
942
943 /* Set our used ptes within the page table */
944 bitmap_set(pt->used_ptes,
945 gen8_pte_index(pd_start),
946 gen8_pte_count(pd_start, pd_len));
947
948 /* Our pde is now pointing to the pagetable, pt */
Mika Kuoppala966082c2015-06-25 18:35:19 +0300949 __set_bit(pde, pd->used_pdes);
Michel Thierryd7b26332015-04-08 12:13:34 +0100950
951 /* Map the PDE to the page table */
Mika Kuoppalafe36f552015-06-25 18:35:16 +0300952 page_directory[pde] = gen8_pde_encode(px_dma(pt),
953 I915_CACHE_LLC);
Michel Thierryd7b26332015-04-08 12:13:34 +0100954
955 /* NB: We haven't yet mapped ptes to pages. At this
956 * point we're still relying on insert_entries() */
Michel Thierry33c88192015-04-08 12:13:33 +0100957 }
Michel Thierryd7b26332015-04-08 12:13:34 +0100958
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +0300959 kunmap_px(ppgtt, page_directory);
Michel Thierryd7b26332015-04-08 12:13:34 +0100960
Mika Kuoppala966082c2015-06-25 18:35:19 +0300961 __set_bit(pdpe, ppgtt->pdp.used_pdpes);
Michel Thierry33c88192015-04-08 12:13:33 +0100962 }
963
Michel Thierryd7b26332015-04-08 12:13:34 +0100964 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
Mika Kuoppala5b7e4c9c2015-06-25 18:35:03 +0300965 mark_tlbs_dirty(ppgtt);
Ben Widawskyd7b3de92015-02-24 16:22:34 +0000966 return 0;
967
968err_out:
Michel Thierryd7b26332015-04-08 12:13:34 +0100969 while (pdpe--) {
970 for_each_set_bit(temp, new_page_tables[pdpe], I915_PDES)
Mika Kuoppalaa08e1112015-06-25 18:35:08 +0300971 free_pt(vm->dev, ppgtt->pdp.page_directory[pdpe]->page_table[temp]);
Michel Thierryd7b26332015-04-08 12:13:34 +0100972 }
973
974 for_each_set_bit(pdpe, new_page_dirs, GEN8_LEGACY_PDPES)
Mika Kuoppalaa08e1112015-06-25 18:35:08 +0300975 free_pd(vm->dev, ppgtt->pdp.page_directory[pdpe]);
Michel Thierryd7b26332015-04-08 12:13:34 +0100976
977 free_gen8_temp_bitmaps(new_page_dirs, new_page_tables);
Mika Kuoppala5b7e4c9c2015-06-25 18:35:03 +0300978 mark_tlbs_dirty(ppgtt);
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800979 return ret;
980}
981
Daniel Vettereb0b44a2015-03-18 14:47:59 +0100982/*
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800983 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
984 * with a net effect resembling a 2-level page table in normal x86 terms. Each
985 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
986 * space.
Ben Widawsky37aca442013-11-04 20:47:32 -0800987 *
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800988 */
Daniel Vetter5c5f6452015-04-14 17:35:14 +0200989static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
Ben Widawsky37aca442013-11-04 20:47:32 -0800990{
Mika Kuoppala79ab9372015-06-25 18:35:17 +0300991 ppgtt->base.scratch_pt = alloc_pt(ppgtt->base.dev);
992 if (IS_ERR(ppgtt->base.scratch_pt))
993 return PTR_ERR(ppgtt->base.scratch_pt);
Michel Thierry69876be2015-04-08 12:13:27 +0100994
Mika Kuoppala79ab9372015-06-25 18:35:17 +0300995 ppgtt->base.scratch_pd = alloc_pd(ppgtt->base.dev);
996 if (IS_ERR(ppgtt->base.scratch_pd))
997 return PTR_ERR(ppgtt->base.scratch_pd);
Michel Thierry7cb6d7a2015-04-08 12:13:29 +0100998
Mika Kuoppala79ab9372015-06-25 18:35:17 +0300999 gen8_initialize_pt(&ppgtt->base, ppgtt->base.scratch_pt);
1000 gen8_initialize_pd(&ppgtt->base, ppgtt->base.scratch_pd);
Michel Thierry69876be2015-04-08 12:13:27 +01001001
Michel Thierryd7b26332015-04-08 12:13:34 +01001002 ppgtt->base.start = 0;
Daniel Vetter5c5f6452015-04-14 17:35:14 +02001003 ppgtt->base.total = 1ULL << 32;
Michel Thierry501fd702015-05-29 14:15:05 +01001004 if (IS_ENABLED(CONFIG_X86_32))
1005 /* While we have a proliferation of size_t variables
1006 * we cannot represent the full ppgtt size on 32bit,
1007 * so limit it to the same size as the GGTT (currently
1008 * 2GiB).
1009 */
1010 ppgtt->base.total = to_i915(ppgtt->base.dev)->gtt.base.total;
Michel Thierryd7b26332015-04-08 12:13:34 +01001011 ppgtt->base.cleanup = gen8_ppgtt_cleanup;
Daniel Vetter5c5f6452015-04-14 17:35:14 +02001012 ppgtt->base.allocate_va_range = gen8_alloc_va_range;
Michel Thierryd7b26332015-04-08 12:13:34 +01001013 ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
Daniel Vetterc7e16f22015-04-14 17:35:11 +02001014 ppgtt->base.clear_range = gen8_ppgtt_clear_range;
Daniel Vetter777dc5b2015-04-14 17:35:12 +02001015 ppgtt->base.unbind_vma = ppgtt_unbind_vma;
1016 ppgtt->base.bind_vma = ppgtt_bind_vma;
Michel Thierryd7b26332015-04-08 12:13:34 +01001017
1018 ppgtt->switch_mm = gen8_mm_switch;
1019
1020 return 0;
1021}
1022
Ben Widawsky87d60b62013-12-06 14:11:29 -08001023static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
1024{
Ben Widawsky87d60b62013-12-06 14:11:29 -08001025 struct i915_address_space *vm = &ppgtt->base;
Michel Thierry09942c62015-04-08 12:13:30 +01001026 struct i915_page_table *unused;
Michel Thierry07749ef2015-03-16 16:00:54 +00001027 gen6_pte_t scratch_pte;
Ben Widawsky87d60b62013-12-06 14:11:29 -08001028 uint32_t pd_entry;
Michel Thierry09942c62015-04-08 12:13:30 +01001029 uint32_t pte, pde, temp;
1030 uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
Ben Widawsky87d60b62013-12-06 14:11:29 -08001031
Mika Kuoppala79ab9372015-06-25 18:35:17 +03001032 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
1033 I915_CACHE_LLC, true, 0);
Ben Widawsky87d60b62013-12-06 14:11:29 -08001034
Michel Thierry09942c62015-04-08 12:13:30 +01001035 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) {
Ben Widawsky87d60b62013-12-06 14:11:29 -08001036 u32 expected;
Michel Thierry07749ef2015-03-16 16:00:54 +00001037 gen6_pte_t *pt_vaddr;
Mika Kuoppala567047b2015-06-25 18:35:12 +03001038 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);
Michel Thierry09942c62015-04-08 12:13:30 +01001039 pd_entry = readl(ppgtt->pd_addr + pde);
Ben Widawsky87d60b62013-12-06 14:11:29 -08001040 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
1041
1042 if (pd_entry != expected)
1043 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
1044 pde,
1045 pd_entry,
1046 expected);
1047 seq_printf(m, "\tPDE: %x\n", pd_entry);
1048
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +03001049 pt_vaddr = kmap_px(ppgtt->pd.page_table[pde]);
1050
Michel Thierry07749ef2015-03-16 16:00:54 +00001051 for (pte = 0; pte < GEN6_PTES; pte+=4) {
Ben Widawsky87d60b62013-12-06 14:11:29 -08001052 unsigned long va =
Michel Thierry07749ef2015-03-16 16:00:54 +00001053 (pde * PAGE_SIZE * GEN6_PTES) +
Ben Widawsky87d60b62013-12-06 14:11:29 -08001054 (pte * PAGE_SIZE);
1055 int i;
1056 bool found = false;
1057 for (i = 0; i < 4; i++)
1058 if (pt_vaddr[pte + i] != scratch_pte)
1059 found = true;
1060 if (!found)
1061 continue;
1062
1063 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
1064 for (i = 0; i < 4; i++) {
1065 if (pt_vaddr[pte + i] != scratch_pte)
1066 seq_printf(m, " %08x", pt_vaddr[pte + i]);
1067 else
1068 seq_puts(m, " SCRATCH ");
1069 }
1070 seq_puts(m, "\n");
1071 }
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +03001072 kunmap_px(ppgtt, pt_vaddr);
Ben Widawsky87d60b62013-12-06 14:11:29 -08001073 }
1074}
1075
Ben Widawsky678d96f2015-03-16 16:00:56 +00001076/* Write pde (index) from the page directory @pd to the page table @pt */
Michel Thierryec565b32015-04-08 12:13:23 +01001077static void gen6_write_pde(struct i915_page_directory *pd,
1078 const int pde, struct i915_page_table *pt)
Ben Widawsky61973492013-04-08 18:43:54 -07001079{
Ben Widawsky678d96f2015-03-16 16:00:56 +00001080 /* Caller needs to make sure the write completes if necessary */
1081 struct i915_hw_ppgtt *ppgtt =
1082 container_of(pd, struct i915_hw_ppgtt, pd);
1083 u32 pd_entry;
Ben Widawsky61973492013-04-08 18:43:54 -07001084
Mika Kuoppala567047b2015-06-25 18:35:12 +03001085 pd_entry = GEN6_PDE_ADDR_ENCODE(px_dma(pt));
Ben Widawsky678d96f2015-03-16 16:00:56 +00001086 pd_entry |= GEN6_PDE_VALID;
Ben Widawsky61973492013-04-08 18:43:54 -07001087
Ben Widawsky678d96f2015-03-16 16:00:56 +00001088 writel(pd_entry, ppgtt->pd_addr + pde);
1089}
Ben Widawsky61973492013-04-08 18:43:54 -07001090
Ben Widawsky678d96f2015-03-16 16:00:56 +00001091/* Write all the page tables found in the ppgtt structure to incrementing page
1092 * directories. */
1093static void gen6_write_page_range(struct drm_i915_private *dev_priv,
Michel Thierryec565b32015-04-08 12:13:23 +01001094 struct i915_page_directory *pd,
Ben Widawsky678d96f2015-03-16 16:00:56 +00001095 uint32_t start, uint32_t length)
1096{
Michel Thierryec565b32015-04-08 12:13:23 +01001097 struct i915_page_table *pt;
Ben Widawsky678d96f2015-03-16 16:00:56 +00001098 uint32_t pde, temp;
1099
1100 gen6_for_each_pde(pt, pd, start, length, temp, pde)
1101 gen6_write_pde(pd, pde, pt);
1102
1103 /* Make sure write is complete before other code can use this page
1104 * table. Also require for WC mapped PTEs */
1105 readl(dev_priv->gtt.gsm);
Ben Widawsky3e302542013-04-23 23:15:32 -07001106}
1107
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001108static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
Ben Widawsky3e302542013-04-23 23:15:32 -07001109{
Mika Kuoppala44159dd2015-06-25 18:35:07 +03001110 BUG_ON(ppgtt->pd.base.ggtt_offset & 0x3f);
Ben Widawsky3e302542013-04-23 23:15:32 -07001111
Mika Kuoppala44159dd2015-06-25 18:35:07 +03001112 return (ppgtt->pd.base.ggtt_offset / 64) << 16;
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001113}
Ben Widawsky61973492013-04-08 18:43:54 -07001114
Ben Widawsky90252e52013-12-06 14:11:12 -08001115static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
John Harrisone85b26d2015-05-29 17:43:56 +01001116 struct drm_i915_gem_request *req)
Ben Widawsky90252e52013-12-06 14:11:12 -08001117{
John Harrisone85b26d2015-05-29 17:43:56 +01001118 struct intel_engine_cs *ring = req->ring;
Ben Widawsky90252e52013-12-06 14:11:12 -08001119 int ret;
Ben Widawsky61973492013-04-08 18:43:54 -07001120
Ben Widawsky90252e52013-12-06 14:11:12 -08001121 /* NB: TLBs must be flushed and invalidated before a switch */
John Harrisona84c3ae2015-05-29 17:43:57 +01001122 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
Ben Widawsky90252e52013-12-06 14:11:12 -08001123 if (ret)
1124 return ret;
1125
John Harrison5fb9de12015-05-29 17:44:07 +01001126 ret = intel_ring_begin(req, 6);
Ben Widawsky90252e52013-12-06 14:11:12 -08001127 if (ret)
1128 return ret;
1129
1130 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1131 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
1132 intel_ring_emit(ring, PP_DIR_DCLV_2G);
1133 intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
1134 intel_ring_emit(ring, get_pd_offset(ppgtt));
1135 intel_ring_emit(ring, MI_NOOP);
1136 intel_ring_advance(ring);
1137
1138 return 0;
1139}
1140
Yu Zhang71ba2d62015-02-10 19:05:54 +08001141static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt,
John Harrisone85b26d2015-05-29 17:43:56 +01001142 struct drm_i915_gem_request *req)
Yu Zhang71ba2d62015-02-10 19:05:54 +08001143{
John Harrisone85b26d2015-05-29 17:43:56 +01001144 struct intel_engine_cs *ring = req->ring;
Yu Zhang71ba2d62015-02-10 19:05:54 +08001145 struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
1146
1147 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
1148 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
1149 return 0;
1150}
1151
Ben Widawsky48a10382013-12-06 14:11:11 -08001152static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
John Harrisone85b26d2015-05-29 17:43:56 +01001153 struct drm_i915_gem_request *req)
Ben Widawsky48a10382013-12-06 14:11:11 -08001154{
John Harrisone85b26d2015-05-29 17:43:56 +01001155 struct intel_engine_cs *ring = req->ring;
Ben Widawsky48a10382013-12-06 14:11:11 -08001156 int ret;
1157
Ben Widawsky48a10382013-12-06 14:11:11 -08001158 /* NB: TLBs must be flushed and invalidated before a switch */
John Harrisona84c3ae2015-05-29 17:43:57 +01001159 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
Ben Widawsky48a10382013-12-06 14:11:11 -08001160 if (ret)
1161 return ret;
1162
John Harrison5fb9de12015-05-29 17:44:07 +01001163 ret = intel_ring_begin(req, 6);
Ben Widawsky48a10382013-12-06 14:11:11 -08001164 if (ret)
1165 return ret;
1166
1167 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
1168 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
1169 intel_ring_emit(ring, PP_DIR_DCLV_2G);
1170 intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
1171 intel_ring_emit(ring, get_pd_offset(ppgtt));
1172 intel_ring_emit(ring, MI_NOOP);
1173 intel_ring_advance(ring);
1174
Ben Widawsky90252e52013-12-06 14:11:12 -08001175 /* XXX: RCS is the only one to auto invalidate the TLBs? */
1176 if (ring->id != RCS) {
John Harrisona84c3ae2015-05-29 17:43:57 +01001177 ret = ring->flush(req, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
Ben Widawsky90252e52013-12-06 14:11:12 -08001178 if (ret)
1179 return ret;
1180 }
1181
Ben Widawsky48a10382013-12-06 14:11:11 -08001182 return 0;
1183}
1184
Ben Widawskyeeb94882013-12-06 14:11:10 -08001185static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
John Harrisone85b26d2015-05-29 17:43:56 +01001186 struct drm_i915_gem_request *req)
Ben Widawskyeeb94882013-12-06 14:11:10 -08001187{
John Harrisone85b26d2015-05-29 17:43:56 +01001188 struct intel_engine_cs *ring = req->ring;
Ben Widawskyeeb94882013-12-06 14:11:10 -08001189 struct drm_device *dev = ppgtt->base.dev;
1190 struct drm_i915_private *dev_priv = dev->dev_private;
1191
Ben Widawsky48a10382013-12-06 14:11:11 -08001192
Ben Widawskyeeb94882013-12-06 14:11:10 -08001193 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
1194 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
1195
1196 POSTING_READ(RING_PP_DIR_DCLV(ring));
1197
1198 return 0;
1199}
1200
Daniel Vetter82460d92014-08-06 20:19:53 +02001201static void gen8_ppgtt_enable(struct drm_device *dev)
Ben Widawskyeeb94882013-12-06 14:11:10 -08001202{
Ben Widawskyeeb94882013-12-06 14:11:10 -08001203 struct drm_i915_private *dev_priv = dev->dev_private;
Oscar Mateoa4872ba2014-05-22 14:13:33 +01001204 struct intel_engine_cs *ring;
Daniel Vetter82460d92014-08-06 20:19:53 +02001205 int j;
Ben Widawskyeeb94882013-12-06 14:11:10 -08001206
1207 for_each_ring(ring, dev_priv, j) {
1208 I915_WRITE(RING_MODE_GEN7(ring),
1209 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
Ben Widawskyeeb94882013-12-06 14:11:10 -08001210 }
Ben Widawskyeeb94882013-12-06 14:11:10 -08001211}
1212
Daniel Vetter82460d92014-08-06 20:19:53 +02001213static void gen7_ppgtt_enable(struct drm_device *dev)
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001214{
Jani Nikula50227e12014-03-31 14:27:21 +03001215 struct drm_i915_private *dev_priv = dev->dev_private;
Oscar Mateoa4872ba2014-05-22 14:13:33 +01001216 struct intel_engine_cs *ring;
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001217 uint32_t ecochk, ecobits;
1218 int i;
1219
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001220 ecobits = I915_READ(GAC_ECO_BITS);
1221 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
1222
1223 ecochk = I915_READ(GAM_ECOCHK);
1224 if (IS_HASWELL(dev)) {
1225 ecochk |= ECOCHK_PPGTT_WB_HSW;
1226 } else {
1227 ecochk |= ECOCHK_PPGTT_LLC_IVB;
1228 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
1229 }
1230 I915_WRITE(GAM_ECOCHK, ecochk);
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001231
Ben Widawsky61973492013-04-08 18:43:54 -07001232 for_each_ring(ring, dev_priv, i) {
Ben Widawskyeeb94882013-12-06 14:11:10 -08001233 /* GFX_MODE is per-ring on gen7+ */
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001234 I915_WRITE(RING_MODE_GEN7(ring),
1235 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
Ben Widawsky61973492013-04-08 18:43:54 -07001236 }
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001237}
1238
Daniel Vetter82460d92014-08-06 20:19:53 +02001239static void gen6_ppgtt_enable(struct drm_device *dev)
Ben Widawsky61973492013-04-08 18:43:54 -07001240{
Jani Nikula50227e12014-03-31 14:27:21 +03001241 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001242 uint32_t ecochk, gab_ctl, ecobits;
Ben Widawsky61973492013-04-08 18:43:54 -07001243
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001244 ecobits = I915_READ(GAC_ECO_BITS);
1245 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
1246 ECOBITS_PPGTT_CACHE64B);
Ben Widawsky61973492013-04-08 18:43:54 -07001247
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001248 gab_ctl = I915_READ(GAB_CTL);
1249 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
Ben Widawsky61973492013-04-08 18:43:54 -07001250
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001251 ecochk = I915_READ(GAM_ECOCHK);
1252 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
Ben Widawsky61973492013-04-08 18:43:54 -07001253
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001254 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
Ben Widawsky61973492013-04-08 18:43:54 -07001255}
1256
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001257/* PPGTT support for Sandybdrige/Gen6 and later */
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001258static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
Ben Widawsky782f1492014-02-20 11:50:33 -08001259 uint64_t start,
1260 uint64_t length,
Ben Widawsky828c7902013-10-16 09:21:30 -07001261 bool use_scratch)
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001262{
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001263 struct i915_hw_ppgtt *ppgtt =
1264 container_of(vm, struct i915_hw_ppgtt, base);
Michel Thierry07749ef2015-03-16 16:00:54 +00001265 gen6_pte_t *pt_vaddr, scratch_pte;
Ben Widawsky782f1492014-02-20 11:50:33 -08001266 unsigned first_entry = start >> PAGE_SHIFT;
1267 unsigned num_entries = length >> PAGE_SHIFT;
Michel Thierry07749ef2015-03-16 16:00:54 +00001268 unsigned act_pt = first_entry / GEN6_PTES;
1269 unsigned first_pte = first_entry % GEN6_PTES;
Daniel Vetter7bddb012012-02-09 17:15:47 +01001270 unsigned last_pte, i;
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001271
Mika Kuoppalac114f762015-06-25 18:35:13 +03001272 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
1273 I915_CACHE_LLC, true, 0);
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001274
Daniel Vetter7bddb012012-02-09 17:15:47 +01001275 while (num_entries) {
1276 last_pte = first_pte + num_entries;
Michel Thierry07749ef2015-03-16 16:00:54 +00001277 if (last_pte > GEN6_PTES)
1278 last_pte = GEN6_PTES;
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001279
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +03001280 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
Daniel Vetter7bddb012012-02-09 17:15:47 +01001281
1282 for (i = first_pte; i < last_pte; i++)
1283 pt_vaddr[i] = scratch_pte;
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001284
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +03001285 kunmap_px(ppgtt, pt_vaddr);
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001286
Daniel Vetter7bddb012012-02-09 17:15:47 +01001287 num_entries -= last_pte - first_pte;
1288 first_pte = 0;
Daniel Vettera15326a2013-03-19 23:48:39 +01001289 act_pt++;
Daniel Vetter7bddb012012-02-09 17:15:47 +01001290 }
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001291}
1292
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001293static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
Daniel Vetterdef886c2013-01-24 14:44:56 -08001294 struct sg_table *pages,
Ben Widawsky782f1492014-02-20 11:50:33 -08001295 uint64_t start,
Akash Goel24f3a8c2014-06-17 10:59:42 +05301296 enum i915_cache_level cache_level, u32 flags)
Daniel Vetterdef886c2013-01-24 14:44:56 -08001297{
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001298 struct i915_hw_ppgtt *ppgtt =
1299 container_of(vm, struct i915_hw_ppgtt, base);
Michel Thierry07749ef2015-03-16 16:00:54 +00001300 gen6_pte_t *pt_vaddr;
Ben Widawsky782f1492014-02-20 11:50:33 -08001301 unsigned first_entry = start >> PAGE_SHIFT;
Michel Thierry07749ef2015-03-16 16:00:54 +00001302 unsigned act_pt = first_entry / GEN6_PTES;
1303 unsigned act_pte = first_entry % GEN6_PTES;
Imre Deak6e995e22013-02-18 19:28:04 +02001304 struct sg_page_iter sg_iter;
Daniel Vetterdef886c2013-01-24 14:44:56 -08001305
Chris Wilsoncc797142013-12-31 15:50:30 +00001306 pt_vaddr = NULL;
Imre Deak6e995e22013-02-18 19:28:04 +02001307 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
Chris Wilsoncc797142013-12-31 15:50:30 +00001308 if (pt_vaddr == NULL)
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +03001309 pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
Daniel Vetterdef886c2013-01-24 14:44:56 -08001310
Chris Wilsoncc797142013-12-31 15:50:30 +00001311 pt_vaddr[act_pte] =
1312 vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
Akash Goel24f3a8c2014-06-17 10:59:42 +05301313 cache_level, true, flags);
1314
Michel Thierry07749ef2015-03-16 16:00:54 +00001315 if (++act_pte == GEN6_PTES) {
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +03001316 kunmap_px(ppgtt, pt_vaddr);
Chris Wilsoncc797142013-12-31 15:50:30 +00001317 pt_vaddr = NULL;
Daniel Vettera15326a2013-03-19 23:48:39 +01001318 act_pt++;
Imre Deak6e995e22013-02-18 19:28:04 +02001319 act_pte = 0;
Daniel Vetterdef886c2013-01-24 14:44:56 -08001320 }
Daniel Vetterdef886c2013-01-24 14:44:56 -08001321 }
Chris Wilsoncc797142013-12-31 15:50:30 +00001322 if (pt_vaddr)
Mika Kuoppalad1c54ac2015-06-25 18:35:11 +03001323 kunmap_px(ppgtt, pt_vaddr);
Daniel Vetterdef886c2013-01-24 14:44:56 -08001324}
1325
Ben Widawsky678d96f2015-03-16 16:00:56 +00001326static int gen6_alloc_va_range(struct i915_address_space *vm,
Mika Kuoppalaa05d80e2015-06-25 18:35:04 +03001327 uint64_t start_in, uint64_t length_in)
Ben Widawsky678d96f2015-03-16 16:00:56 +00001328{
Michel Thierry4933d512015-03-24 15:46:22 +00001329 DECLARE_BITMAP(new_page_tables, I915_PDES);
1330 struct drm_device *dev = vm->dev;
1331 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawsky678d96f2015-03-16 16:00:56 +00001332 struct i915_hw_ppgtt *ppgtt =
1333 container_of(vm, struct i915_hw_ppgtt, base);
Michel Thierryec565b32015-04-08 12:13:23 +01001334 struct i915_page_table *pt;
Mika Kuoppalaa05d80e2015-06-25 18:35:04 +03001335 uint32_t start, length, start_save, length_save;
Ben Widawsky678d96f2015-03-16 16:00:56 +00001336 uint32_t pde, temp;
Michel Thierry4933d512015-03-24 15:46:22 +00001337 int ret;
1338
Mika Kuoppalaa05d80e2015-06-25 18:35:04 +03001339 if (WARN_ON(start_in + length_in > ppgtt->base.total))
1340 return -ENODEV;
1341
1342 start = start_save = start_in;
1343 length = length_save = length_in;
Michel Thierry4933d512015-03-24 15:46:22 +00001344
1345 bitmap_zero(new_page_tables, I915_PDES);
1346
1347 /* The allocation is done in two stages so that we can bail out with
1348 * minimal amount of pain. The first stage finds new page tables that
1349 * need allocation. The second stage marks use ptes within the page
1350 * tables.
1351 */
1352 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
Mika Kuoppala79ab9372015-06-25 18:35:17 +03001353 if (pt != vm->scratch_pt) {
Michel Thierry4933d512015-03-24 15:46:22 +00001354 WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES));
1355 continue;
1356 }
1357
1358 /* We've already allocated a page table */
1359 WARN_ON(!bitmap_empty(pt->used_ptes, GEN6_PTES));
1360
Mika Kuoppala8a1ebd72015-05-22 20:04:59 +03001361 pt = alloc_pt(dev);
Michel Thierry4933d512015-03-24 15:46:22 +00001362 if (IS_ERR(pt)) {
1363 ret = PTR_ERR(pt);
1364 goto unwind_out;
1365 }
1366
1367 gen6_initialize_pt(vm, pt);
1368
1369 ppgtt->pd.page_table[pde] = pt;
Mika Kuoppala966082c2015-06-25 18:35:19 +03001370 __set_bit(pde, new_page_tables);
Michel Thierry72744cb2015-03-24 15:46:23 +00001371 trace_i915_page_table_entry_alloc(vm, pde, start, GEN6_PDE_SHIFT);
Michel Thierry4933d512015-03-24 15:46:22 +00001372 }
1373
1374 start = start_save;
1375 length = length_save;
Ben Widawsky678d96f2015-03-16 16:00:56 +00001376
1377 gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
1378 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES);
1379
1380 bitmap_zero(tmp_bitmap, GEN6_PTES);
1381 bitmap_set(tmp_bitmap, gen6_pte_index(start),
1382 gen6_pte_count(start, length));
1383
Mika Kuoppala966082c2015-06-25 18:35:19 +03001384 if (__test_and_clear_bit(pde, new_page_tables))
Michel Thierry4933d512015-03-24 15:46:22 +00001385 gen6_write_pde(&ppgtt->pd, pde, pt);
1386
Michel Thierry72744cb2015-03-24 15:46:23 +00001387 trace_i915_page_table_entry_map(vm, pde, pt,
1388 gen6_pte_index(start),
1389 gen6_pte_count(start, length),
1390 GEN6_PTES);
Michel Thierry4933d512015-03-24 15:46:22 +00001391 bitmap_or(pt->used_ptes, tmp_bitmap, pt->used_ptes,
Ben Widawsky678d96f2015-03-16 16:00:56 +00001392 GEN6_PTES);
1393 }
1394
Michel Thierry4933d512015-03-24 15:46:22 +00001395 WARN_ON(!bitmap_empty(new_page_tables, I915_PDES));
1396
1397 /* Make sure write is complete before other code can use this page
1398 * table. Also require for WC mapped PTEs */
1399 readl(dev_priv->gtt.gsm);
1400
Ben Widawsky563222a2015-03-19 12:53:28 +00001401 mark_tlbs_dirty(ppgtt);
Ben Widawsky678d96f2015-03-16 16:00:56 +00001402 return 0;
Michel Thierry4933d512015-03-24 15:46:22 +00001403
1404unwind_out:
1405 for_each_set_bit(pde, new_page_tables, I915_PDES) {
Michel Thierryec565b32015-04-08 12:13:23 +01001406 struct i915_page_table *pt = ppgtt->pd.page_table[pde];
Michel Thierry4933d512015-03-24 15:46:22 +00001407
Mika Kuoppala79ab9372015-06-25 18:35:17 +03001408 ppgtt->pd.page_table[pde] = vm->scratch_pt;
Mika Kuoppalaa08e1112015-06-25 18:35:08 +03001409 free_pt(vm->dev, pt);
Michel Thierry4933d512015-03-24 15:46:22 +00001410 }
1411
1412 mark_tlbs_dirty(ppgtt);
1413 return ret;
Ben Widawsky678d96f2015-03-16 16:00:56 +00001414}
1415
Daniel Vetter061dd492015-04-14 17:35:13 +02001416static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
Ben Widawskya00d8252014-02-19 22:05:48 -08001417{
Daniel Vetter061dd492015-04-14 17:35:13 +02001418 struct i915_hw_ppgtt *ppgtt =
1419 container_of(vm, struct i915_hw_ppgtt, base);
Michel Thierry09942c62015-04-08 12:13:30 +01001420 struct i915_page_table *pt;
1421 uint32_t pde;
Daniel Vetter3440d262013-01-24 13:49:56 -08001422
Daniel Vetter061dd492015-04-14 17:35:13 +02001423 drm_mm_remove_node(&ppgtt->node);
1424
Michel Thierry09942c62015-04-08 12:13:30 +01001425 gen6_for_all_pdes(pt, ppgtt, pde) {
Mika Kuoppala79ab9372015-06-25 18:35:17 +03001426 if (pt != vm->scratch_pt)
Mika Kuoppalaa08e1112015-06-25 18:35:08 +03001427 free_pt(ppgtt->base.dev, pt);
Michel Thierry4933d512015-03-24 15:46:22 +00001428 }
1429
Mika Kuoppala79ab9372015-06-25 18:35:17 +03001430 free_pt(vm->dev, vm->scratch_pt);
Daniel Vetter3440d262013-01-24 13:49:56 -08001431}
1432
Ben Widawskyb1465202014-02-19 22:05:49 -08001433static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
Daniel Vetter3440d262013-01-24 13:49:56 -08001434{
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001435 struct drm_device *dev = ppgtt->base.dev;
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001436 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawskye3cc1992013-12-06 14:11:08 -08001437 bool retried = false;
Ben Widawskyb1465202014-02-19 22:05:49 -08001438 int ret;
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001439
Ben Widawskyc8d4c0d2013-12-06 14:11:07 -08001440 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
1441 * allocator works in address space sizes, so it's multiplied by page
1442 * size. We allocate at the top of the GTT to avoid fragmentation.
1443 */
1444 BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
Mika Kuoppala79ab9372015-06-25 18:35:17 +03001445 ppgtt->base.scratch_pt = alloc_pt(ppgtt->base.dev);
1446 if (IS_ERR(ppgtt->base.scratch_pt))
1447 return PTR_ERR(ppgtt->base.scratch_pt);
Michel Thierry4933d512015-03-24 15:46:22 +00001448
Mika Kuoppala79ab9372015-06-25 18:35:17 +03001449 gen6_initialize_pt(&ppgtt->base, ppgtt->base.scratch_pt);
Michel Thierry4933d512015-03-24 15:46:22 +00001450
Ben Widawskye3cc1992013-12-06 14:11:08 -08001451alloc:
Ben Widawskyc8d4c0d2013-12-06 14:11:07 -08001452 ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
1453 &ppgtt->node, GEN6_PD_SIZE,
1454 GEN6_PD_ALIGN, 0,
1455 0, dev_priv->gtt.base.total,
Ben Widawsky3e8b5ae2014-05-06 22:21:30 -07001456 DRM_MM_TOPDOWN);
Ben Widawskye3cc1992013-12-06 14:11:08 -08001457 if (ret == -ENOSPC && !retried) {
1458 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
1459 GEN6_PD_SIZE, GEN6_PD_ALIGN,
Chris Wilsond23db882014-05-23 08:48:08 +02001460 I915_CACHE_NONE,
1461 0, dev_priv->gtt.base.total,
1462 0);
Ben Widawskye3cc1992013-12-06 14:11:08 -08001463 if (ret)
Ben Widawsky678d96f2015-03-16 16:00:56 +00001464 goto err_out;
Ben Widawskye3cc1992013-12-06 14:11:08 -08001465
1466 retried = true;
1467 goto alloc;
1468 }
Ben Widawskyc8d4c0d2013-12-06 14:11:07 -08001469
Ben Widawskyc8c26622015-01-22 17:01:25 +00001470 if (ret)
Ben Widawsky678d96f2015-03-16 16:00:56 +00001471 goto err_out;
1472
Ben Widawskyc8c26622015-01-22 17:01:25 +00001473
Ben Widawskyc8d4c0d2013-12-06 14:11:07 -08001474 if (ppgtt->node.start < dev_priv->gtt.mappable_end)
1475 DRM_DEBUG("Forced to use aperture for PDEs\n");
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001476
Ben Widawskyc8c26622015-01-22 17:01:25 +00001477 return 0;
Ben Widawsky678d96f2015-03-16 16:00:56 +00001478
1479err_out:
Mika Kuoppala79ab9372015-06-25 18:35:17 +03001480 free_pt(ppgtt->base.dev, ppgtt->base.scratch_pt);
Ben Widawsky678d96f2015-03-16 16:00:56 +00001481 return ret;
Ben Widawskyb1465202014-02-19 22:05:49 -08001482}
1483
Ben Widawskyb1465202014-02-19 22:05:49 -08001484static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
1485{
kbuild test robot2f2cf682015-03-27 19:26:35 +08001486 return gen6_ppgtt_allocate_page_directories(ppgtt);
Ben Widawskyb1465202014-02-19 22:05:49 -08001487}
1488
Michel Thierry4933d512015-03-24 15:46:22 +00001489static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
1490 uint64_t start, uint64_t length)
1491{
Michel Thierryec565b32015-04-08 12:13:23 +01001492 struct i915_page_table *unused;
Michel Thierry4933d512015-03-24 15:46:22 +00001493 uint32_t pde, temp;
1494
1495 gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde)
Mika Kuoppala79ab9372015-06-25 18:35:17 +03001496 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;
Michel Thierry4933d512015-03-24 15:46:22 +00001497}
1498
Daniel Vetter5c5f6452015-04-14 17:35:14 +02001499static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
Ben Widawskyb1465202014-02-19 22:05:49 -08001500{
1501 struct drm_device *dev = ppgtt->base.dev;
1502 struct drm_i915_private *dev_priv = dev->dev_private;
1503 int ret;
1504
1505 ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
Ben Widawsky48a10382013-12-06 14:11:11 -08001506 if (IS_GEN6(dev)) {
Ben Widawsky48a10382013-12-06 14:11:11 -08001507 ppgtt->switch_mm = gen6_mm_switch;
Ben Widawsky90252e52013-12-06 14:11:12 -08001508 } else if (IS_HASWELL(dev)) {
Ben Widawsky90252e52013-12-06 14:11:12 -08001509 ppgtt->switch_mm = hsw_mm_switch;
Ben Widawsky48a10382013-12-06 14:11:11 -08001510 } else if (IS_GEN7(dev)) {
Ben Widawsky48a10382013-12-06 14:11:11 -08001511 ppgtt->switch_mm = gen7_mm_switch;
1512 } else
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001513 BUG();
Ben Widawskyb1465202014-02-19 22:05:49 -08001514
Yu Zhang71ba2d62015-02-10 19:05:54 +08001515 if (intel_vgpu_active(dev))
1516 ppgtt->switch_mm = vgpu_mm_switch;
1517
Ben Widawskyb1465202014-02-19 22:05:49 -08001518 ret = gen6_ppgtt_alloc(ppgtt);
1519 if (ret)
1520 return ret;
1521
Daniel Vetter5c5f6452015-04-14 17:35:14 +02001522 ppgtt->base.allocate_va_range = gen6_alloc_va_range;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001523 ppgtt->base.clear_range = gen6_ppgtt_clear_range;
1524 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
Daniel Vetter777dc5b2015-04-14 17:35:12 +02001525 ppgtt->base.unbind_vma = ppgtt_unbind_vma;
1526 ppgtt->base.bind_vma = ppgtt_bind_vma;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001527 ppgtt->base.cleanup = gen6_ppgtt_cleanup;
Ben Widawsky686e1f6f2013-11-25 09:54:34 -08001528 ppgtt->base.start = 0;
Michel Thierry09942c62015-04-08 12:13:30 +01001529 ppgtt->base.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
Ben Widawskyb1465202014-02-19 22:05:49 -08001530 ppgtt->debug_dump = gen6_dump_ppgtt;
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001531
Mika Kuoppala44159dd2015-06-25 18:35:07 +03001532 ppgtt->pd.base.ggtt_offset =
Michel Thierry07749ef2015-03-16 16:00:54 +00001533 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_pte_t);
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001534
Ben Widawsky678d96f2015-03-16 16:00:56 +00001535 ppgtt->pd_addr = (gen6_pte_t __iomem *)dev_priv->gtt.gsm +
Mika Kuoppala44159dd2015-06-25 18:35:07 +03001536 ppgtt->pd.base.ggtt_offset / sizeof(gen6_pte_t);
Ben Widawsky678d96f2015-03-16 16:00:56 +00001537
Daniel Vetter5c5f6452015-04-14 17:35:14 +02001538 gen6_scratch_va_range(ppgtt, 0, ppgtt->base.total);
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001539
Ben Widawsky678d96f2015-03-16 16:00:56 +00001540 gen6_write_page_range(dev_priv, &ppgtt->pd, 0, ppgtt->base.total);
1541
Thierry Reding440fd522015-01-23 09:05:06 +01001542 DRM_DEBUG_DRIVER("Allocated pde space (%lldM) at GTT entry: %llx\n",
Ben Widawskyc8d4c0d2013-12-06 14:11:07 -08001543 ppgtt->node.size >> 20,
1544 ppgtt->node.start / PAGE_SIZE);
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001545
Daniel Vetterfa76da32014-08-06 20:19:54 +02001546 DRM_DEBUG("Adding PPGTT at offset %x\n",
Mika Kuoppala44159dd2015-06-25 18:35:07 +03001547 ppgtt->pd.base.ggtt_offset << 10);
Daniel Vetterfa76da32014-08-06 20:19:54 +02001548
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001549 return 0;
Daniel Vetter3440d262013-01-24 13:49:56 -08001550}
1551
Daniel Vetter5c5f6452015-04-14 17:35:14 +02001552static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
Daniel Vetter3440d262013-01-24 13:49:56 -08001553{
1554 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter3440d262013-01-24 13:49:56 -08001555
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001556 ppgtt->base.dev = dev;
Mika Kuoppalac114f762015-06-25 18:35:13 +03001557 ppgtt->base.scratch_page = dev_priv->gtt.base.scratch_page;
Daniel Vetter3440d262013-01-24 13:49:56 -08001558
Ben Widawsky3ed124b2013-04-08 18:43:53 -07001559 if (INTEL_INFO(dev)->gen < 8)
Daniel Vetter5c5f6452015-04-14 17:35:14 +02001560 return gen6_ppgtt_init(ppgtt);
Ben Widawsky3ed124b2013-04-08 18:43:53 -07001561 else
Michel Thierryd7b26332015-04-08 12:13:34 +01001562 return gen8_ppgtt_init(ppgtt);
Daniel Vetterfa76da32014-08-06 20:19:54 +02001563}
Mika Kuoppalac114f762015-06-25 18:35:13 +03001564
Daniel Vetterfa76da32014-08-06 20:19:54 +02001565int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
1566{
1567 struct drm_i915_private *dev_priv = dev->dev_private;
1568 int ret = 0;
Ben Widawsky3ed124b2013-04-08 18:43:53 -07001569
Daniel Vetter5c5f6452015-04-14 17:35:14 +02001570 ret = __hw_ppgtt_init(dev, ppgtt);
Daniel Vetterfa76da32014-08-06 20:19:54 +02001571 if (ret == 0) {
Ben Widawskyc7c48df2013-12-06 14:11:15 -08001572 kref_init(&ppgtt->ref);
Ben Widawsky93bd8642013-07-16 16:50:06 -07001573 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
1574 ppgtt->base.total);
Ben Widawsky7e0d96b2013-12-06 14:11:26 -08001575 i915_init_vm(dev_priv, &ppgtt->base);
Ben Widawsky93bd8642013-07-16 16:50:06 -07001576 }
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001577
1578 return ret;
1579}
1580
Daniel Vetter82460d92014-08-06 20:19:53 +02001581int i915_ppgtt_init_hw(struct drm_device *dev)
1582{
Thomas Daniel671b50132014-08-20 16:24:50 +01001583 /* In the case of execlists, PPGTT is enabled by the context descriptor
1584 * and the PDPs are contained within the context itself. We don't
1585 * need to do anything here. */
1586 if (i915.enable_execlists)
1587 return 0;
1588
Daniel Vetter82460d92014-08-06 20:19:53 +02001589 if (!USES_PPGTT(dev))
1590 return 0;
1591
1592 if (IS_GEN6(dev))
1593 gen6_ppgtt_enable(dev);
1594 else if (IS_GEN7(dev))
1595 gen7_ppgtt_enable(dev);
1596 else if (INTEL_INFO(dev)->gen >= 8)
1597 gen8_ppgtt_enable(dev);
1598 else
Daniel Vetter5f77eeb2014-12-08 16:40:10 +01001599 MISSING_CASE(INTEL_INFO(dev)->gen);
Daniel Vetter82460d92014-08-06 20:19:53 +02001600
John Harrison4ad2fd82015-06-18 13:11:20 +01001601 return 0;
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001602}
John Harrison4ad2fd82015-06-18 13:11:20 +01001603
John Harrisonb3dd6b92015-05-29 17:43:40 +01001604int i915_ppgtt_init_ring(struct drm_i915_gem_request *req)
John Harrison4ad2fd82015-06-18 13:11:20 +01001605{
John Harrisonb3dd6b92015-05-29 17:43:40 +01001606 struct drm_i915_private *dev_priv = req->ring->dev->dev_private;
John Harrison4ad2fd82015-06-18 13:11:20 +01001607 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
1608
1609 if (i915.enable_execlists)
1610 return 0;
1611
1612 if (!ppgtt)
1613 return 0;
1614
John Harrisone85b26d2015-05-29 17:43:56 +01001615 return ppgtt->switch_mm(ppgtt, req);
John Harrison4ad2fd82015-06-18 13:11:20 +01001616}
1617
Daniel Vetter4d884702014-08-06 15:04:47 +02001618struct i915_hw_ppgtt *
1619i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
1620{
1621 struct i915_hw_ppgtt *ppgtt;
1622 int ret;
1623
1624 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
1625 if (!ppgtt)
1626 return ERR_PTR(-ENOMEM);
1627
1628 ret = i915_ppgtt_init(dev, ppgtt);
1629 if (ret) {
1630 kfree(ppgtt);
1631 return ERR_PTR(ret);
1632 }
1633
1634 ppgtt->file_priv = fpriv;
1635
Daniele Ceraolo Spurio198c9742014-11-10 13:44:31 +00001636 trace_i915_ppgtt_create(&ppgtt->base);
1637
Daniel Vetter4d884702014-08-06 15:04:47 +02001638 return ppgtt;
1639}
1640
Daniel Vetteree960be2014-08-06 15:04:45 +02001641void i915_ppgtt_release(struct kref *kref)
1642{
1643 struct i915_hw_ppgtt *ppgtt =
1644 container_of(kref, struct i915_hw_ppgtt, ref);
1645
Daniele Ceraolo Spurio198c9742014-11-10 13:44:31 +00001646 trace_i915_ppgtt_release(&ppgtt->base);
1647
Daniel Vetteree960be2014-08-06 15:04:45 +02001648 /* vmas should already be unbound */
1649 WARN_ON(!list_empty(&ppgtt->base.active_list));
1650 WARN_ON(!list_empty(&ppgtt->base.inactive_list));
1651
Daniel Vetter19dd1202014-08-06 15:04:55 +02001652 list_del(&ppgtt->base.global_link);
1653 drm_mm_takedown(&ppgtt->base.mm);
1654
Daniel Vetteree960be2014-08-06 15:04:45 +02001655 ppgtt->base.cleanup(&ppgtt->base);
1656 kfree(ppgtt);
1657}
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001658
Ben Widawskya81cc002013-01-18 12:30:31 -08001659extern int intel_iommu_gfx_mapped;
1660/* Certain Gen5 chipsets require require idling the GPU before
1661 * unmapping anything from the GTT when VT-d is enabled.
1662 */
Daniel Vetter2c642b02015-04-14 17:35:26 +02001663static bool needs_idle_maps(struct drm_device *dev)
Ben Widawskya81cc002013-01-18 12:30:31 -08001664{
1665#ifdef CONFIG_INTEL_IOMMU
1666 /* Query intel_iommu to see if we need the workaround. Presumably that
1667 * was loaded first.
1668 */
1669 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
1670 return true;
1671#endif
1672 return false;
1673}
1674
Ben Widawsky5c042282011-10-17 15:51:55 -07001675static bool do_idling(struct drm_i915_private *dev_priv)
1676{
1677 bool ret = dev_priv->mm.interruptible;
1678
Ben Widawskya81cc002013-01-18 12:30:31 -08001679 if (unlikely(dev_priv->gtt.do_idle_maps)) {
Ben Widawsky5c042282011-10-17 15:51:55 -07001680 dev_priv->mm.interruptible = false;
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07001681 if (i915_gpu_idle(dev_priv->dev)) {
Ben Widawsky5c042282011-10-17 15:51:55 -07001682 DRM_ERROR("Couldn't idle GPU\n");
1683 /* Wait a bit, in hopes it avoids the hang */
1684 udelay(10);
1685 }
1686 }
1687
1688 return ret;
1689}
1690
1691static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
1692{
Ben Widawskya81cc002013-01-18 12:30:31 -08001693 if (unlikely(dev_priv->gtt.do_idle_maps))
Ben Widawsky5c042282011-10-17 15:51:55 -07001694 dev_priv->mm.interruptible = interruptible;
1695}
1696
Ben Widawsky828c7902013-10-16 09:21:30 -07001697void i915_check_and_clear_faults(struct drm_device *dev)
1698{
1699 struct drm_i915_private *dev_priv = dev->dev_private;
Oscar Mateoa4872ba2014-05-22 14:13:33 +01001700 struct intel_engine_cs *ring;
Ben Widawsky828c7902013-10-16 09:21:30 -07001701 int i;
1702
1703 if (INTEL_INFO(dev)->gen < 6)
1704 return;
1705
1706 for_each_ring(ring, dev_priv, i) {
1707 u32 fault_reg;
1708 fault_reg = I915_READ(RING_FAULT_REG(ring));
1709 if (fault_reg & RING_FAULT_VALID) {
1710 DRM_DEBUG_DRIVER("Unexpected fault\n"
Paulo Zanoni59a5d292014-10-30 15:52:45 -02001711 "\tAddr: 0x%08lx\n"
Ben Widawsky828c7902013-10-16 09:21:30 -07001712 "\tAddress space: %s\n"
1713 "\tSource ID: %d\n"
1714 "\tType: %d\n",
1715 fault_reg & PAGE_MASK,
1716 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
1717 RING_FAULT_SRCID(fault_reg),
1718 RING_FAULT_FAULT_TYPE(fault_reg));
1719 I915_WRITE(RING_FAULT_REG(ring),
1720 fault_reg & ~RING_FAULT_VALID);
1721 }
1722 }
1723 POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
1724}
1725
Chris Wilson91e56492014-09-25 10:13:12 +01001726static void i915_ggtt_flush(struct drm_i915_private *dev_priv)
1727{
1728 if (INTEL_INFO(dev_priv->dev)->gen < 6) {
1729 intel_gtt_chipset_flush();
1730 } else {
1731 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1732 POSTING_READ(GFX_FLSH_CNTL_GEN6);
1733 }
1734}
1735
Ben Widawsky828c7902013-10-16 09:21:30 -07001736void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
1737{
1738 struct drm_i915_private *dev_priv = dev->dev_private;
1739
1740 /* Don't bother messing with faults pre GEN6 as we have little
1741 * documentation supporting that it's a good idea.
1742 */
1743 if (INTEL_INFO(dev)->gen < 6)
1744 return;
1745
1746 i915_check_and_clear_faults(dev);
1747
1748 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
Ben Widawsky782f1492014-02-20 11:50:33 -08001749 dev_priv->gtt.base.start,
1750 dev_priv->gtt.base.total,
Daniel Vettere568af12014-03-26 20:08:20 +01001751 true);
Chris Wilson91e56492014-09-25 10:13:12 +01001752
1753 i915_ggtt_flush(dev_priv);
Ben Widawsky828c7902013-10-16 09:21:30 -07001754}
1755
Daniel Vetter74163902012-02-15 23:50:21 +01001756int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01001757{
Chris Wilson9da3da62012-06-01 15:20:22 +01001758 if (obj->has_dma_mapping)
Daniel Vetter74163902012-02-15 23:50:21 +01001759 return 0;
Chris Wilson9da3da62012-06-01 15:20:22 +01001760
1761 if (!dma_map_sg(&obj->base.dev->pdev->dev,
1762 obj->pages->sgl, obj->pages->nents,
1763 PCI_DMA_BIDIRECTIONAL))
1764 return -ENOSPC;
1765
1766 return 0;
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01001767}
1768
Daniel Vetter2c642b02015-04-14 17:35:26 +02001769static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001770{
1771#ifdef writeq
1772 writeq(pte, addr);
1773#else
1774 iowrite32((u32)pte, addr);
1775 iowrite32(pte >> 32, addr + 4);
1776#endif
1777}
1778
1779static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
1780 struct sg_table *st,
Ben Widawsky782f1492014-02-20 11:50:33 -08001781 uint64_t start,
Akash Goel24f3a8c2014-06-17 10:59:42 +05301782 enum i915_cache_level level, u32 unused)
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001783{
1784 struct drm_i915_private *dev_priv = vm->dev->dev_private;
Ben Widawsky782f1492014-02-20 11:50:33 -08001785 unsigned first_entry = start >> PAGE_SHIFT;
Michel Thierry07749ef2015-03-16 16:00:54 +00001786 gen8_pte_t __iomem *gtt_entries =
1787 (gen8_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001788 int i = 0;
1789 struct sg_page_iter sg_iter;
Pavel Machek57007df2014-07-28 13:20:58 +02001790 dma_addr_t addr = 0; /* shut up gcc */
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001791
1792 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1793 addr = sg_dma_address(sg_iter.sg) +
1794 (sg_iter.sg_pgoffset << PAGE_SHIFT);
1795 gen8_set_pte(&gtt_entries[i],
1796 gen8_pte_encode(addr, level, true));
1797 i++;
1798 }
1799
1800 /*
1801 * XXX: This serves as a posting read to make sure that the PTE has
1802 * actually been updated. There is some concern that even though
1803 * registers and PTEs are within the same BAR that they are potentially
1804 * of NUMA access patterns. Therefore, even with the way we assume
1805 * hardware should work, we must keep this posting read for paranoia.
1806 */
1807 if (i != 0)
1808 WARN_ON(readq(&gtt_entries[i-1])
1809 != gen8_pte_encode(addr, level, true));
1810
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001811 /* This next bit makes the above posting read even more important. We
1812 * want to flush the TLBs only after we're certain all the PTE updates
1813 * have finished.
1814 */
1815 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1816 POSTING_READ(GFX_FLSH_CNTL_GEN6);
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001817}
1818
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001819/*
1820 * Binds an object into the global gtt with the specified cache level. The object
1821 * will be accessible to the GPU via commands whose operands reference offsets
1822 * within the global GTT as well as accessible by the GPU through the GMADR
1823 * mapped BAR (dev_priv->mm.gtt->gtt).
1824 */
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001825static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001826 struct sg_table *st,
Ben Widawsky782f1492014-02-20 11:50:33 -08001827 uint64_t start,
Akash Goel24f3a8c2014-06-17 10:59:42 +05301828 enum i915_cache_level level, u32 flags)
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001829{
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001830 struct drm_i915_private *dev_priv = vm->dev->dev_private;
Ben Widawsky782f1492014-02-20 11:50:33 -08001831 unsigned first_entry = start >> PAGE_SHIFT;
Michel Thierry07749ef2015-03-16 16:00:54 +00001832 gen6_pte_t __iomem *gtt_entries =
1833 (gen6_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
Imre Deak6e995e22013-02-18 19:28:04 +02001834 int i = 0;
1835 struct sg_page_iter sg_iter;
Pavel Machek57007df2014-07-28 13:20:58 +02001836 dma_addr_t addr = 0;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001837
Imre Deak6e995e22013-02-18 19:28:04 +02001838 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
Imre Deak2db76d72013-03-26 15:14:18 +02001839 addr = sg_page_iter_dma_address(&sg_iter);
Akash Goel24f3a8c2014-06-17 10:59:42 +05301840 iowrite32(vm->pte_encode(addr, level, true, flags), &gtt_entries[i]);
Imre Deak6e995e22013-02-18 19:28:04 +02001841 i++;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001842 }
1843
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001844 /* XXX: This serves as a posting read to make sure that the PTE has
1845 * actually been updated. There is some concern that even though
1846 * registers and PTEs are within the same BAR that they are potentially
1847 * of NUMA access patterns. Therefore, even with the way we assume
1848 * hardware should work, we must keep this posting read for paranoia.
1849 */
Pavel Machek57007df2014-07-28 13:20:58 +02001850 if (i != 0) {
1851 unsigned long gtt = readl(&gtt_entries[i-1]);
1852 WARN_ON(gtt != vm->pte_encode(addr, level, true, flags));
1853 }
Ben Widawsky0f9b91c2012-11-04 09:21:30 -08001854
1855 /* This next bit makes the above posting read even more important. We
1856 * want to flush the TLBs only after we're certain all the PTE updates
1857 * have finished.
1858 */
1859 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1860 POSTING_READ(GFX_FLSH_CNTL_GEN6);
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001861}
1862
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001863static void gen8_ggtt_clear_range(struct i915_address_space *vm,
Ben Widawsky782f1492014-02-20 11:50:33 -08001864 uint64_t start,
1865 uint64_t length,
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001866 bool use_scratch)
1867{
1868 struct drm_i915_private *dev_priv = vm->dev->dev_private;
Ben Widawsky782f1492014-02-20 11:50:33 -08001869 unsigned first_entry = start >> PAGE_SHIFT;
1870 unsigned num_entries = length >> PAGE_SHIFT;
Michel Thierry07749ef2015-03-16 16:00:54 +00001871 gen8_pte_t scratch_pte, __iomem *gtt_base =
1872 (gen8_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001873 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1874 int i;
1875
1876 if (WARN(num_entries > max_entries,
1877 "First entry = %d; Num entries = %d (max=%d)\n",
1878 first_entry, num_entries, max_entries))
1879 num_entries = max_entries;
1880
Mika Kuoppalac114f762015-06-25 18:35:13 +03001881 scratch_pte = gen8_pte_encode(px_dma(vm->scratch_page),
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001882 I915_CACHE_LLC,
1883 use_scratch);
1884 for (i = 0; i < num_entries; i++)
1885 gen8_set_pte(&gtt_base[i], scratch_pte);
1886 readl(gtt_base);
1887}
1888
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001889static void gen6_ggtt_clear_range(struct i915_address_space *vm,
Ben Widawsky782f1492014-02-20 11:50:33 -08001890 uint64_t start,
1891 uint64_t length,
Ben Widawsky828c7902013-10-16 09:21:30 -07001892 bool use_scratch)
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001893{
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001894 struct drm_i915_private *dev_priv = vm->dev->dev_private;
Ben Widawsky782f1492014-02-20 11:50:33 -08001895 unsigned first_entry = start >> PAGE_SHIFT;
1896 unsigned num_entries = length >> PAGE_SHIFT;
Michel Thierry07749ef2015-03-16 16:00:54 +00001897 gen6_pte_t scratch_pte, __iomem *gtt_base =
1898 (gen6_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
Ben Widawskya54c0c22013-01-24 14:45:00 -08001899 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001900 int i;
1901
1902 if (WARN(num_entries > max_entries,
1903 "First entry = %d; Num entries = %d (max=%d)\n",
1904 first_entry, num_entries, max_entries))
1905 num_entries = max_entries;
1906
Mika Kuoppalac114f762015-06-25 18:35:13 +03001907 scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
1908 I915_CACHE_LLC, use_scratch, 0);
Ben Widawsky828c7902013-10-16 09:21:30 -07001909
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001910 for (i = 0; i < num_entries; i++)
1911 iowrite32(scratch_pte, &gtt_base[i]);
1912 readl(gtt_base);
1913}
1914
Daniel Vetterd369d2d2015-04-14 17:35:25 +02001915static void i915_ggtt_insert_entries(struct i915_address_space *vm,
1916 struct sg_table *pages,
1917 uint64_t start,
1918 enum i915_cache_level cache_level, u32 unused)
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001919{
1920 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
1921 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
1922
Daniel Vetterd369d2d2015-04-14 17:35:25 +02001923 intel_gtt_insert_sg_entries(pages, start >> PAGE_SHIFT, flags);
Daniel Vetter08755462015-04-20 09:04:05 -07001924
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001925}
1926
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001927static void i915_ggtt_clear_range(struct i915_address_space *vm,
Ben Widawsky782f1492014-02-20 11:50:33 -08001928 uint64_t start,
1929 uint64_t length,
Ben Widawsky828c7902013-10-16 09:21:30 -07001930 bool unused)
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001931{
Ben Widawsky782f1492014-02-20 11:50:33 -08001932 unsigned first_entry = start >> PAGE_SHIFT;
1933 unsigned num_entries = length >> PAGE_SHIFT;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001934 intel_gtt_clear_range(first_entry, num_entries);
1935}
1936
Daniel Vetter70b9f6f2015-04-14 17:35:27 +02001937static int ggtt_bind_vma(struct i915_vma *vma,
1938 enum i915_cache_level cache_level,
1939 u32 flags)
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01001940{
Ben Widawsky6f65e292013-12-06 14:10:56 -08001941 struct drm_device *dev = vma->vm->dev;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001942 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawsky6f65e292013-12-06 14:10:56 -08001943 struct drm_i915_gem_object *obj = vma->obj;
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02001944 struct sg_table *pages = obj->pages;
Daniel Vetterf329f5f2015-04-14 17:35:15 +02001945 u32 pte_flags = 0;
Daniel Vetter70b9f6f2015-04-14 17:35:27 +02001946 int ret;
1947
1948 ret = i915_get_ggtt_vma_pages(vma);
1949 if (ret)
1950 return ret;
1951 pages = vma->ggtt_view.pages;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001952
Akash Goel24f3a8c2014-06-17 10:59:42 +05301953 /* Currently applicable only to VLV */
1954 if (obj->gt_ro)
Daniel Vetterf329f5f2015-04-14 17:35:15 +02001955 pte_flags |= PTE_READ_ONLY;
Akash Goel24f3a8c2014-06-17 10:59:42 +05301956
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02001957
Ben Widawsky6f65e292013-12-06 14:10:56 -08001958 if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
Daniel Vetter08755462015-04-20 09:04:05 -07001959 vma->vm->insert_entries(vma->vm, pages,
1960 vma->node.start,
1961 cache_level, pte_flags);
Ben Widawsky6f65e292013-12-06 14:10:56 -08001962 }
Daniel Vetter74898d72012-02-15 23:50:22 +01001963
Daniel Vetter08755462015-04-20 09:04:05 -07001964 if (dev_priv->mm.aliasing_ppgtt && flags & LOCAL_BIND) {
Ben Widawsky6f65e292013-12-06 14:10:56 -08001965 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02001966 appgtt->base.insert_entries(&appgtt->base, pages,
Ben Widawsky782f1492014-02-20 11:50:33 -08001967 vma->node.start,
Daniel Vetterf329f5f2015-04-14 17:35:15 +02001968 cache_level, pte_flags);
Ben Widawsky6f65e292013-12-06 14:10:56 -08001969 }
Daniel Vetter70b9f6f2015-04-14 17:35:27 +02001970
1971 return 0;
Ben Widawsky6f65e292013-12-06 14:10:56 -08001972}
1973
1974static void ggtt_unbind_vma(struct i915_vma *vma)
1975{
1976 struct drm_device *dev = vma->vm->dev;
1977 struct drm_i915_private *dev_priv = dev->dev_private;
1978 struct drm_i915_gem_object *obj = vma->obj;
Joonas Lahtinen06615ee2015-04-24 15:09:03 +03001979 const uint64_t size = min_t(uint64_t,
1980 obj->base.size,
1981 vma->node.size);
Ben Widawsky6f65e292013-12-06 14:10:56 -08001982
Tvrtko Ursulinaff43762014-10-24 12:42:33 +01001983 if (vma->bound & GLOBAL_BIND) {
Ben Widawsky782f1492014-02-20 11:50:33 -08001984 vma->vm->clear_range(vma->vm,
1985 vma->node.start,
Joonas Lahtinen06615ee2015-04-24 15:09:03 +03001986 size,
Ben Widawsky6f65e292013-12-06 14:10:56 -08001987 true);
Ben Widawsky6f65e292013-12-06 14:10:56 -08001988 }
1989
Daniel Vetter08755462015-04-20 09:04:05 -07001990 if (dev_priv->mm.aliasing_ppgtt && vma->bound & LOCAL_BIND) {
Ben Widawsky6f65e292013-12-06 14:10:56 -08001991 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
Joonas Lahtinen06615ee2015-04-24 15:09:03 +03001992
Ben Widawsky6f65e292013-12-06 14:10:56 -08001993 appgtt->base.clear_range(&appgtt->base,
Ben Widawsky782f1492014-02-20 11:50:33 -08001994 vma->node.start,
Joonas Lahtinen06615ee2015-04-24 15:09:03 +03001995 size,
Ben Widawsky6f65e292013-12-06 14:10:56 -08001996 true);
Ben Widawsky6f65e292013-12-06 14:10:56 -08001997 }
Daniel Vetter74163902012-02-15 23:50:21 +01001998}
1999
2000void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
2001{
Ben Widawsky5c042282011-10-17 15:51:55 -07002002 struct drm_device *dev = obj->base.dev;
2003 struct drm_i915_private *dev_priv = dev->dev_private;
2004 bool interruptible;
2005
2006 interruptible = do_idling(dev_priv);
2007
Chris Wilson9da3da62012-06-01 15:20:22 +01002008 if (!obj->has_dma_mapping)
2009 dma_unmap_sg(&dev->pdev->dev,
2010 obj->pages->sgl, obj->pages->nents,
2011 PCI_DMA_BIDIRECTIONAL);
Ben Widawsky5c042282011-10-17 15:51:55 -07002012
2013 undo_idling(dev_priv, interruptible);
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01002014}
Daniel Vetter644ec022012-03-26 09:45:40 +02002015
Chris Wilson42d6ab42012-07-26 11:49:32 +01002016static void i915_gtt_color_adjust(struct drm_mm_node *node,
2017 unsigned long color,
Thierry Reding440fd522015-01-23 09:05:06 +01002018 u64 *start,
2019 u64 *end)
Chris Wilson42d6ab42012-07-26 11:49:32 +01002020{
2021 if (node->color != color)
2022 *start += 4096;
2023
2024 if (!list_empty(&node->node_list)) {
2025 node = list_entry(node->node_list.next,
2026 struct drm_mm_node,
2027 node_list);
2028 if (node->allocated && node->color != color)
2029 *end -= 4096;
2030 }
2031}
Ben Widawskyfbe5d362013-11-04 19:56:49 -08002032
Daniel Vetterf548c0e2014-11-19 21:40:13 +01002033static int i915_gem_setup_global_gtt(struct drm_device *dev,
2034 unsigned long start,
2035 unsigned long mappable_end,
2036 unsigned long end)
Daniel Vetter644ec022012-03-26 09:45:40 +02002037{
Ben Widawskye78891c2013-01-25 16:41:04 -08002038 /* Let GEM Manage all of the aperture.
2039 *
2040 * However, leave one page at the end still bound to the scratch page.
2041 * There are a number of places where the hardware apparently prefetches
2042 * past the end of the object, and we've seen multiple hangs with the
2043 * GPU head pointer stuck in a batchbuffer bound at the last page of the
2044 * aperture. One page should be enough to keep any prefetching inside
2045 * of the aperture.
2046 */
Ben Widawsky40d749802013-07-31 16:59:59 -07002047 struct drm_i915_private *dev_priv = dev->dev_private;
2048 struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
Chris Wilsoned2f3452012-11-15 11:32:19 +00002049 struct drm_mm_node *entry;
2050 struct drm_i915_gem_object *obj;
2051 unsigned long hole_start, hole_end;
Daniel Vetterfa76da32014-08-06 20:19:54 +02002052 int ret;
Daniel Vetter644ec022012-03-26 09:45:40 +02002053
Ben Widawsky35451cb2013-01-17 12:45:13 -08002054 BUG_ON(mappable_end > end);
2055
Chris Wilsoned2f3452012-11-15 11:32:19 +00002056 /* Subtract the guard page ... */
Ben Widawsky40d749802013-07-31 16:59:59 -07002057 drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
Yu Zhang5dda8fa2015-02-10 19:05:48 +08002058
2059 dev_priv->gtt.base.start = start;
2060 dev_priv->gtt.base.total = end - start;
2061
2062 if (intel_vgpu_active(dev)) {
2063 ret = intel_vgt_balloon(dev);
2064 if (ret)
2065 return ret;
2066 }
2067
Chris Wilson42d6ab42012-07-26 11:49:32 +01002068 if (!HAS_LLC(dev))
Ben Widawsky93bd8642013-07-16 16:50:06 -07002069 dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
Daniel Vetter644ec022012-03-26 09:45:40 +02002070
Chris Wilsoned2f3452012-11-15 11:32:19 +00002071 /* Mark any preallocated objects as occupied */
Ben Widawsky35c20a62013-05-31 11:28:48 -07002072 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
Ben Widawsky40d749802013-07-31 16:59:59 -07002073 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
Daniel Vetterfa76da32014-08-06 20:19:54 +02002074
Ben Widawskyedd41a82013-07-05 14:41:05 -07002075 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
Ben Widawskyc6cfb322013-07-05 14:41:06 -07002076 i915_gem_obj_ggtt_offset(obj), obj->base.size);
Chris Wilsoned2f3452012-11-15 11:32:19 +00002077
Ben Widawskyc6cfb322013-07-05 14:41:06 -07002078 WARN_ON(i915_gem_obj_ggtt_bound(obj));
Ben Widawsky40d749802013-07-31 16:59:59 -07002079 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
Daniel Vetter6c5566a2014-08-06 15:04:50 +02002080 if (ret) {
2081 DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
2082 return ret;
2083 }
Tvrtko Ursulinaff43762014-10-24 12:42:33 +01002084 vma->bound |= GLOBAL_BIND;
Chris Wilsoned2f3452012-11-15 11:32:19 +00002085 }
2086
Chris Wilsoned2f3452012-11-15 11:32:19 +00002087 /* Clear any non-preallocated blocks */
Ben Widawsky40d749802013-07-31 16:59:59 -07002088 drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
Chris Wilsoned2f3452012-11-15 11:32:19 +00002089 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
2090 hole_start, hole_end);
Ben Widawsky782f1492014-02-20 11:50:33 -08002091 ggtt_vm->clear_range(ggtt_vm, hole_start,
2092 hole_end - hole_start, true);
Chris Wilsoned2f3452012-11-15 11:32:19 +00002093 }
2094
2095 /* And finally clear the reserved guard page */
Ben Widawsky782f1492014-02-20 11:50:33 -08002096 ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
Daniel Vetter6c5566a2014-08-06 15:04:50 +02002097
Daniel Vetterfa76da32014-08-06 20:19:54 +02002098 if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
2099 struct i915_hw_ppgtt *ppgtt;
2100
2101 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
2102 if (!ppgtt)
2103 return -ENOMEM;
2104
Daniel Vetter5c5f6452015-04-14 17:35:14 +02002105 ret = __hw_ppgtt_init(dev, ppgtt);
Michel Thierry4933d512015-03-24 15:46:22 +00002106 if (ret) {
Daniel Vetter061dd492015-04-14 17:35:13 +02002107 ppgtt->base.cleanup(&ppgtt->base);
Michel Thierry4933d512015-03-24 15:46:22 +00002108 kfree(ppgtt);
Daniel Vetterfa76da32014-08-06 20:19:54 +02002109 return ret;
Michel Thierry4933d512015-03-24 15:46:22 +00002110 }
Daniel Vetterfa76da32014-08-06 20:19:54 +02002111
Daniel Vetter5c5f6452015-04-14 17:35:14 +02002112 if (ppgtt->base.allocate_va_range)
2113 ret = ppgtt->base.allocate_va_range(&ppgtt->base, 0,
2114 ppgtt->base.total);
2115 if (ret) {
2116 ppgtt->base.cleanup(&ppgtt->base);
2117 kfree(ppgtt);
2118 return ret;
2119 }
2120
2121 ppgtt->base.clear_range(&ppgtt->base,
2122 ppgtt->base.start,
2123 ppgtt->base.total,
2124 true);
2125
Daniel Vetterfa76da32014-08-06 20:19:54 +02002126 dev_priv->mm.aliasing_ppgtt = ppgtt;
2127 }
2128
Daniel Vetter6c5566a2014-08-06 15:04:50 +02002129 return 0;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002130}
2131
Ben Widawskyd7e50082012-12-18 10:31:25 -08002132void i915_gem_init_global_gtt(struct drm_device *dev)
2133{
2134 struct drm_i915_private *dev_priv = dev->dev_private;
Mika Kuoppalac44ef602015-06-25 18:35:05 +03002135 u64 gtt_size, mappable_size;
Ben Widawskyd7e50082012-12-18 10:31:25 -08002136
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002137 gtt_size = dev_priv->gtt.base.total;
Ben Widawsky93d18792013-01-17 12:45:17 -08002138 mappable_size = dev_priv->gtt.mappable_end;
Ben Widawskyd7e50082012-12-18 10:31:25 -08002139
Ben Widawskye78891c2013-01-25 16:41:04 -08002140 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002141}
2142
Daniel Vetter90d0a0e2014-08-06 15:04:56 +02002143void i915_global_gtt_cleanup(struct drm_device *dev)
2144{
2145 struct drm_i915_private *dev_priv = dev->dev_private;
2146 struct i915_address_space *vm = &dev_priv->gtt.base;
2147
Daniel Vetter70e32542014-08-06 15:04:57 +02002148 if (dev_priv->mm.aliasing_ppgtt) {
2149 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
2150
2151 ppgtt->base.cleanup(&ppgtt->base);
2152 }
2153
Daniel Vetter90d0a0e2014-08-06 15:04:56 +02002154 if (drm_mm_initialized(&vm->mm)) {
Yu Zhang5dda8fa2015-02-10 19:05:48 +08002155 if (intel_vgpu_active(dev))
2156 intel_vgt_deballoon();
2157
Daniel Vetter90d0a0e2014-08-06 15:04:56 +02002158 drm_mm_takedown(&vm->mm);
2159 list_del(&vm->global_link);
2160 }
2161
2162 vm->cleanup(vm);
2163}
Daniel Vetter70e32542014-08-06 15:04:57 +02002164
Daniel Vetter2c642b02015-04-14 17:35:26 +02002165static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002166{
2167 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
2168 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
2169 return snb_gmch_ctl << 20;
2170}
2171
Daniel Vetter2c642b02015-04-14 17:35:26 +02002172static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
Ben Widawsky9459d252013-11-03 16:53:55 -08002173{
2174 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
2175 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
2176 if (bdw_gmch_ctl)
2177 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
Ben Widawsky562d55d2014-05-27 16:53:08 -07002178
2179#ifdef CONFIG_X86_32
2180 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * PAGE_SIZE */
2181 if (bdw_gmch_ctl > 4)
2182 bdw_gmch_ctl = 4;
2183#endif
2184
Ben Widawsky9459d252013-11-03 16:53:55 -08002185 return bdw_gmch_ctl << 20;
2186}
2187
Daniel Vetter2c642b02015-04-14 17:35:26 +02002188static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
Damien Lespiaud7f25f22014-05-08 22:19:40 +03002189{
2190 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
2191 gmch_ctrl &= SNB_GMCH_GGMS_MASK;
2192
2193 if (gmch_ctrl)
2194 return 1 << (20 + gmch_ctrl);
2195
2196 return 0;
2197}
2198
Daniel Vetter2c642b02015-04-14 17:35:26 +02002199static size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002200{
2201 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
2202 snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
2203 return snb_gmch_ctl << 25; /* 32 MB units */
2204}
2205
Daniel Vetter2c642b02015-04-14 17:35:26 +02002206static size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
Ben Widawsky9459d252013-11-03 16:53:55 -08002207{
2208 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2209 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
2210 return bdw_gmch_ctl << 25; /* 32 MB units */
2211}
2212
Damien Lespiaud7f25f22014-05-08 22:19:40 +03002213static size_t chv_get_stolen_size(u16 gmch_ctrl)
2214{
2215 gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
2216 gmch_ctrl &= SNB_GMCH_GMS_MASK;
2217
2218 /*
2219 * 0x0 to 0x10: 32MB increments starting at 0MB
2220 * 0x11 to 0x16: 4MB increments starting at 8MB
2221 * 0x17 to 0x1d: 4MB increments start at 36MB
2222 */
2223 if (gmch_ctrl < 0x11)
2224 return gmch_ctrl << 25;
2225 else if (gmch_ctrl < 0x17)
2226 return (gmch_ctrl - 0x11 + 2) << 22;
2227 else
2228 return (gmch_ctrl - 0x17 + 9) << 22;
2229}
2230
Damien Lespiau66375012014-01-09 18:02:46 +00002231static size_t gen9_get_stolen_size(u16 gen9_gmch_ctl)
2232{
2233 gen9_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
2234 gen9_gmch_ctl &= BDW_GMCH_GMS_MASK;
2235
2236 if (gen9_gmch_ctl < 0xf0)
2237 return gen9_gmch_ctl << 25; /* 32 MB units */
2238 else
2239 /* 4MB increments starting at 0xf0 for 4MB */
2240 return (gen9_gmch_ctl - 0xf0 + 1) << 22;
2241}
2242
Ben Widawsky63340132013-11-04 19:32:22 -08002243static int ggtt_probe_common(struct drm_device *dev,
2244 size_t gtt_size)
2245{
2246 struct drm_i915_private *dev_priv = dev->dev_private;
Bjorn Helgaas21c34602013-12-21 10:52:52 -07002247 phys_addr_t gtt_phys_addr;
Ben Widawsky63340132013-11-04 19:32:22 -08002248 int ret;
2249
2250 /* For Modern GENs the PTEs and register space are split in the BAR */
Bjorn Helgaas21c34602013-12-21 10:52:52 -07002251 gtt_phys_addr = pci_resource_start(dev->pdev, 0) +
Ben Widawsky63340132013-11-04 19:32:22 -08002252 (pci_resource_len(dev->pdev, 0) / 2);
2253
Imre Deak2a073f892015-03-27 13:07:33 +02002254 /*
2255 * On BXT writes larger than 64 bit to the GTT pagetable range will be
2256 * dropped. For WC mappings in general we have 64 byte burst writes
2257 * when the WC buffer is flushed, so we can't use it, but have to
2258 * resort to an uncached mapping. The WC issue is easily caught by the
2259 * readback check when writing GTT PTE entries.
2260 */
2261 if (IS_BROXTON(dev))
2262 dev_priv->gtt.gsm = ioremap_nocache(gtt_phys_addr, gtt_size);
2263 else
2264 dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size);
Ben Widawsky63340132013-11-04 19:32:22 -08002265 if (!dev_priv->gtt.gsm) {
2266 DRM_ERROR("Failed to map the gtt page table\n");
2267 return -ENOMEM;
2268 }
2269
Mika Kuoppalac114f762015-06-25 18:35:13 +03002270 ret = alloc_scratch_page(&dev_priv->gtt.base);
Ben Widawsky63340132013-11-04 19:32:22 -08002271 if (ret) {
2272 DRM_ERROR("Scratch setup failed\n");
2273 /* iounmap will also get called at remove, but meh */
2274 iounmap(dev_priv->gtt.gsm);
2275 }
2276
2277 return ret;
2278}
2279
Ben Widawskyfbe5d362013-11-04 19:56:49 -08002280/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
2281 * bits. When using advanced contexts each context stores its own PAT, but
2282 * writing this data shouldn't be harmful even in those cases. */
Ville Syrjäläee0ce472014-04-09 13:28:01 +03002283static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
Ben Widawskyfbe5d362013-11-04 19:56:49 -08002284{
Ben Widawskyfbe5d362013-11-04 19:56:49 -08002285 uint64_t pat;
2286
2287 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */
2288 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
2289 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
2290 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */
2291 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
2292 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
2293 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
2294 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
2295
Rodrigo Vivid6a8b722014-11-05 16:56:36 -08002296 if (!USES_PPGTT(dev_priv->dev))
2297 /* Spec: "For GGTT, there is NO pat_sel[2:0] from the entry,
2298 * so RTL will always use the value corresponding to
2299 * pat_sel = 000".
2300 * So let's disable cache for GGTT to avoid screen corruptions.
2301 * MOCS still can be used though.
2302 * - System agent ggtt writes (i.e. cpu gtt mmaps) already work
2303 * before this patch, i.e. the same uncached + snooping access
2304 * like on gen6/7 seems to be in effect.
2305 * - So this just fixes blitter/render access. Again it looks
2306 * like it's not just uncached access, but uncached + snooping.
2307 * So we can still hold onto all our assumptions wrt cpu
2308 * clflushing on LLC machines.
2309 */
2310 pat = GEN8_PPAT(0, GEN8_PPAT_UC);
2311
Ben Widawskyfbe5d362013-11-04 19:56:49 -08002312 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
2313 * write would work. */
2314 I915_WRITE(GEN8_PRIVATE_PAT, pat);
2315 I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
2316}
2317
Ville Syrjäläee0ce472014-04-09 13:28:01 +03002318static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
2319{
2320 uint64_t pat;
2321
2322 /*
2323 * Map WB on BDW to snooped on CHV.
2324 *
2325 * Only the snoop bit has meaning for CHV, the rest is
2326 * ignored.
2327 *
Ville Syrjäläcf3d2622014-11-14 21:02:44 +02002328 * The hardware will never snoop for certain types of accesses:
2329 * - CPU GTT (GMADR->GGTT->no snoop->memory)
2330 * - PPGTT page tables
2331 * - some other special cycles
2332 *
2333 * As with BDW, we also need to consider the following for GT accesses:
2334 * "For GGTT, there is NO pat_sel[2:0] from the entry,
2335 * so RTL will always use the value corresponding to
2336 * pat_sel = 000".
2337 * Which means we must set the snoop bit in PAT entry 0
2338 * in order to keep the global status page working.
Ville Syrjäläee0ce472014-04-09 13:28:01 +03002339 */
2340 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
2341 GEN8_PPAT(1, 0) |
2342 GEN8_PPAT(2, 0) |
2343 GEN8_PPAT(3, 0) |
2344 GEN8_PPAT(4, CHV_PPAT_SNOOP) |
2345 GEN8_PPAT(5, CHV_PPAT_SNOOP) |
2346 GEN8_PPAT(6, CHV_PPAT_SNOOP) |
2347 GEN8_PPAT(7, CHV_PPAT_SNOOP);
2348
2349 I915_WRITE(GEN8_PRIVATE_PAT, pat);
2350 I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
2351}
2352
Ben Widawsky63340132013-11-04 19:32:22 -08002353static int gen8_gmch_probe(struct drm_device *dev,
Mika Kuoppalac44ef602015-06-25 18:35:05 +03002354 u64 *gtt_total,
Ben Widawsky63340132013-11-04 19:32:22 -08002355 size_t *stolen,
2356 phys_addr_t *mappable_base,
Mika Kuoppalac44ef602015-06-25 18:35:05 +03002357 u64 *mappable_end)
Ben Widawsky63340132013-11-04 19:32:22 -08002358{
2359 struct drm_i915_private *dev_priv = dev->dev_private;
Mika Kuoppalac44ef602015-06-25 18:35:05 +03002360 u64 gtt_size;
Ben Widawsky63340132013-11-04 19:32:22 -08002361 u16 snb_gmch_ctl;
2362 int ret;
2363
2364 /* TODO: We're not aware of mappable constraints on gen8 yet */
2365 *mappable_base = pci_resource_start(dev->pdev, 2);
2366 *mappable_end = pci_resource_len(dev->pdev, 2);
2367
2368 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
2369 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
2370
2371 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
2372
Damien Lespiau66375012014-01-09 18:02:46 +00002373 if (INTEL_INFO(dev)->gen >= 9) {
2374 *stolen = gen9_get_stolen_size(snb_gmch_ctl);
2375 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
2376 } else if (IS_CHERRYVIEW(dev)) {
Damien Lespiaud7f25f22014-05-08 22:19:40 +03002377 *stolen = chv_get_stolen_size(snb_gmch_ctl);
2378 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl);
2379 } else {
2380 *stolen = gen8_get_stolen_size(snb_gmch_ctl);
2381 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
2382 }
Ben Widawsky63340132013-11-04 19:32:22 -08002383
Michel Thierry07749ef2015-03-16 16:00:54 +00002384 *gtt_total = (gtt_size / sizeof(gen8_pte_t)) << PAGE_SHIFT;
Ben Widawsky63340132013-11-04 19:32:22 -08002385
Sumit Singh5a4e33a2015-03-17 11:39:31 +02002386 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
Ville Syrjäläee0ce472014-04-09 13:28:01 +03002387 chv_setup_private_ppat(dev_priv);
2388 else
2389 bdw_setup_private_ppat(dev_priv);
Ben Widawskyfbe5d362013-11-04 19:56:49 -08002390
Ben Widawsky63340132013-11-04 19:32:22 -08002391 ret = ggtt_probe_common(dev, gtt_size);
2392
Ben Widawsky94ec8f62013-11-02 21:07:18 -07002393 dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
2394 dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
Daniel Vetter777dc5b2015-04-14 17:35:12 +02002395 dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
2396 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
Ben Widawsky63340132013-11-04 19:32:22 -08002397
2398 return ret;
2399}
2400
Ben Widawskybaa09f52013-01-24 13:49:57 -08002401static int gen6_gmch_probe(struct drm_device *dev,
Mika Kuoppalac44ef602015-06-25 18:35:05 +03002402 u64 *gtt_total,
Ben Widawsky41907dd2013-02-08 11:32:47 -08002403 size_t *stolen,
2404 phys_addr_t *mappable_base,
Mika Kuoppalac44ef602015-06-25 18:35:05 +03002405 u64 *mappable_end)
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002406{
2407 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawskybaa09f52013-01-24 13:49:57 -08002408 unsigned int gtt_size;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002409 u16 snb_gmch_ctl;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002410 int ret;
2411
Ben Widawsky41907dd2013-02-08 11:32:47 -08002412 *mappable_base = pci_resource_start(dev->pdev, 2);
2413 *mappable_end = pci_resource_len(dev->pdev, 2);
2414
Ben Widawskybaa09f52013-01-24 13:49:57 -08002415 /* 64/512MB is the current min/max we actually know of, but this is just
2416 * a coarse sanity check.
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002417 */
Ben Widawsky41907dd2013-02-08 11:32:47 -08002418 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
Mika Kuoppalac44ef602015-06-25 18:35:05 +03002419 DRM_ERROR("Unknown GMADR size (%llx)\n",
Ben Widawskybaa09f52013-01-24 13:49:57 -08002420 dev_priv->gtt.mappable_end);
2421 return -ENXIO;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002422 }
2423
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002424 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
2425 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
Ben Widawskybaa09f52013-01-24 13:49:57 -08002426 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
Ben Widawskybaa09f52013-01-24 13:49:57 -08002427
Ben Widawskyc4ae25e2013-05-01 11:00:34 -07002428 *stolen = gen6_get_stolen_size(snb_gmch_ctl);
Ben Widawskybaa09f52013-01-24 13:49:57 -08002429
Ben Widawsky63340132013-11-04 19:32:22 -08002430 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
Michel Thierry07749ef2015-03-16 16:00:54 +00002431 *gtt_total = (gtt_size / sizeof(gen6_pte_t)) << PAGE_SHIFT;
Ben Widawskybaa09f52013-01-24 13:49:57 -08002432
Ben Widawsky63340132013-11-04 19:32:22 -08002433 ret = ggtt_probe_common(dev, gtt_size);
Ben Widawskybaa09f52013-01-24 13:49:57 -08002434
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002435 dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
2436 dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
Daniel Vetter777dc5b2015-04-14 17:35:12 +02002437 dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
2438 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
Ben Widawskybaa09f52013-01-24 13:49:57 -08002439
2440 return ret;
2441}
2442
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002443static void gen6_gmch_remove(struct i915_address_space *vm)
Ben Widawskybaa09f52013-01-24 13:49:57 -08002444{
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002445
2446 struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
Ben Widawsky5ed16782013-11-25 09:54:43 -08002447
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002448 iounmap(gtt->gsm);
Mika Kuoppalac114f762015-06-25 18:35:13 +03002449 free_scratch_page(vm);
Ben Widawskybaa09f52013-01-24 13:49:57 -08002450}
2451
2452static int i915_gmch_probe(struct drm_device *dev,
Mika Kuoppalac44ef602015-06-25 18:35:05 +03002453 u64 *gtt_total,
Ben Widawsky41907dd2013-02-08 11:32:47 -08002454 size_t *stolen,
2455 phys_addr_t *mappable_base,
Mika Kuoppalac44ef602015-06-25 18:35:05 +03002456 u64 *mappable_end)
Ben Widawskybaa09f52013-01-24 13:49:57 -08002457{
2458 struct drm_i915_private *dev_priv = dev->dev_private;
2459 int ret;
2460
Ben Widawskybaa09f52013-01-24 13:49:57 -08002461 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
2462 if (!ret) {
2463 DRM_ERROR("failed to set up gmch\n");
2464 return -EIO;
2465 }
2466
Ben Widawsky41907dd2013-02-08 11:32:47 -08002467 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
Ben Widawskybaa09f52013-01-24 13:49:57 -08002468
2469 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
Daniel Vetterd369d2d2015-04-14 17:35:25 +02002470 dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002471 dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
Daniel Vetterd369d2d2015-04-14 17:35:25 +02002472 dev_priv->gtt.base.bind_vma = ggtt_bind_vma;
2473 dev_priv->gtt.base.unbind_vma = ggtt_unbind_vma;
Ben Widawskybaa09f52013-01-24 13:49:57 -08002474
Chris Wilsonc0a7f812013-12-30 12:16:15 +00002475 if (unlikely(dev_priv->gtt.do_idle_maps))
2476 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
2477
Ben Widawskybaa09f52013-01-24 13:49:57 -08002478 return 0;
2479}
2480
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002481static void i915_gmch_remove(struct i915_address_space *vm)
Ben Widawskybaa09f52013-01-24 13:49:57 -08002482{
2483 intel_gmch_remove();
2484}
2485
2486int i915_gem_gtt_init(struct drm_device *dev)
2487{
2488 struct drm_i915_private *dev_priv = dev->dev_private;
2489 struct i915_gtt *gtt = &dev_priv->gtt;
Ben Widawskybaa09f52013-01-24 13:49:57 -08002490 int ret;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002491
Ben Widawskybaa09f52013-01-24 13:49:57 -08002492 if (INTEL_INFO(dev)->gen <= 5) {
Ben Widawskyb2f21b42013-06-27 16:30:20 -07002493 gtt->gtt_probe = i915_gmch_probe;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002494 gtt->base.cleanup = i915_gmch_remove;
Ben Widawsky63340132013-11-04 19:32:22 -08002495 } else if (INTEL_INFO(dev)->gen < 8) {
Ben Widawskyb2f21b42013-06-27 16:30:20 -07002496 gtt->gtt_probe = gen6_gmch_probe;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002497 gtt->base.cleanup = gen6_gmch_remove;
Ben Widawsky4d15c142013-07-04 11:02:06 -07002498 if (IS_HASWELL(dev) && dev_priv->ellc_size)
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002499 gtt->base.pte_encode = iris_pte_encode;
Ben Widawsky4d15c142013-07-04 11:02:06 -07002500 else if (IS_HASWELL(dev))
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002501 gtt->base.pte_encode = hsw_pte_encode;
Ben Widawskyb2f21b42013-06-27 16:30:20 -07002502 else if (IS_VALLEYVIEW(dev))
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002503 gtt->base.pte_encode = byt_pte_encode;
Chris Wilson350ec882013-08-06 13:17:02 +01002504 else if (INTEL_INFO(dev)->gen >= 7)
2505 gtt->base.pte_encode = ivb_pte_encode;
Ben Widawskyb2f21b42013-06-27 16:30:20 -07002506 else
Chris Wilson350ec882013-08-06 13:17:02 +01002507 gtt->base.pte_encode = snb_pte_encode;
Ben Widawsky63340132013-11-04 19:32:22 -08002508 } else {
2509 dev_priv->gtt.gtt_probe = gen8_gmch_probe;
2510 dev_priv->gtt.base.cleanup = gen6_gmch_remove;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002511 }
2512
Mika Kuoppalac114f762015-06-25 18:35:13 +03002513 gtt->base.dev = dev;
2514
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002515 ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
Ben Widawskyb2f21b42013-06-27 16:30:20 -07002516 &gtt->mappable_base, &gtt->mappable_end);
Ben Widawskya54c0c22013-01-24 14:45:00 -08002517 if (ret)
Ben Widawskybaa09f52013-01-24 13:49:57 -08002518 return ret;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002519
Ben Widawskybaa09f52013-01-24 13:49:57 -08002520 /* GMADR is the PCI mmio aperture into the global GTT. */
Mika Kuoppalac44ef602015-06-25 18:35:05 +03002521 DRM_INFO("Memory usable by graphics device = %lluM\n",
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002522 gtt->base.total >> 20);
Mika Kuoppalac44ef602015-06-25 18:35:05 +03002523 DRM_DEBUG_DRIVER("GMADR size = %lldM\n", gtt->mappable_end >> 20);
Ben Widawskyb2f21b42013-06-27 16:30:20 -07002524 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
Daniel Vetter5db6c732014-03-31 16:23:04 +02002525#ifdef CONFIG_INTEL_IOMMU
2526 if (intel_iommu_gfx_mapped)
2527 DRM_INFO("VT-d active for gfx access\n");
2528#endif
Daniel Vettercfa7c862014-04-29 11:53:58 +02002529 /*
2530 * i915.enable_ppgtt is read-only, so do an early pass to validate the
2531 * user's requested state against the hardware/driver capabilities. We
2532 * do this now so that we can print out any log messages once rather
2533 * than every time we check intel_enable_ppgtt().
2534 */
2535 i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt);
2536 DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08002537
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002538 return 0;
Daniel Vetter644ec022012-03-26 09:45:40 +02002539}
Ben Widawsky6f65e292013-12-06 14:10:56 -08002540
Daniel Vetterfa423312015-04-14 17:35:23 +02002541void i915_gem_restore_gtt_mappings(struct drm_device *dev)
2542{
2543 struct drm_i915_private *dev_priv = dev->dev_private;
2544 struct drm_i915_gem_object *obj;
2545 struct i915_address_space *vm;
2546
2547 i915_check_and_clear_faults(dev);
2548
2549 /* First fill our portion of the GTT with scratch pages */
2550 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
2551 dev_priv->gtt.base.start,
2552 dev_priv->gtt.base.total,
2553 true);
2554
2555 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
2556 struct i915_vma *vma = i915_gem_obj_to_vma(obj,
2557 &dev_priv->gtt.base);
2558 if (!vma)
2559 continue;
2560
2561 i915_gem_clflush_object(obj, obj->pin_display);
2562 WARN_ON(i915_vma_bind(vma, obj->cache_level, PIN_UPDATE));
2563 }
2564
2565
2566 if (INTEL_INFO(dev)->gen >= 8) {
2567 if (IS_CHERRYVIEW(dev) || IS_BROXTON(dev))
2568 chv_setup_private_ppat(dev_priv);
2569 else
2570 bdw_setup_private_ppat(dev_priv);
2571
2572 return;
2573 }
2574
2575 if (USES_PPGTT(dev)) {
2576 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
2577 /* TODO: Perhaps it shouldn't be gen6 specific */
2578
2579 struct i915_hw_ppgtt *ppgtt =
2580 container_of(vm, struct i915_hw_ppgtt,
2581 base);
2582
2583 if (i915_is_ggtt(vm))
2584 ppgtt = dev_priv->mm.aliasing_ppgtt;
2585
2586 gen6_write_page_range(dev_priv, &ppgtt->pd,
2587 0, ppgtt->base.total);
2588 }
2589 }
2590
2591 i915_ggtt_flush(dev_priv);
2592}
2593
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02002594static struct i915_vma *
2595__i915_gem_vma_create(struct drm_i915_gem_object *obj,
2596 struct i915_address_space *vm,
2597 const struct i915_ggtt_view *ggtt_view)
Ben Widawsky6f65e292013-12-06 14:10:56 -08002598{
Dan Carpenterdabde5c2015-03-18 11:21:58 +03002599 struct i915_vma *vma;
Ben Widawsky6f65e292013-12-06 14:10:56 -08002600
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02002601 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
2602 return ERR_PTR(-EINVAL);
Chris Wilsone20d2ab2015-04-07 16:20:58 +01002603
2604 vma = kmem_cache_zalloc(to_i915(obj->base.dev)->vmas, GFP_KERNEL);
Dan Carpenterdabde5c2015-03-18 11:21:58 +03002605 if (vma == NULL)
2606 return ERR_PTR(-ENOMEM);
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02002607
Ben Widawsky6f65e292013-12-06 14:10:56 -08002608 INIT_LIST_HEAD(&vma->vma_link);
2609 INIT_LIST_HEAD(&vma->mm_list);
2610 INIT_LIST_HEAD(&vma->exec_list);
2611 vma->vm = vm;
2612 vma->obj = obj;
2613
Daniel Vetter777dc5b2015-04-14 17:35:12 +02002614 if (i915_is_ggtt(vm))
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02002615 vma->ggtt_view = *ggtt_view;
Ben Widawsky6f65e292013-12-06 14:10:56 -08002616
Tvrtko Ursulinf7635662014-12-03 14:59:24 +00002617 list_add_tail(&vma->vma_link, &obj->vma_list);
2618 if (!i915_is_ggtt(vm))
Michel Thierrye07f0552014-08-19 15:49:41 +01002619 i915_ppgtt_get(i915_vm_to_ppgtt(vm));
Ben Widawsky6f65e292013-12-06 14:10:56 -08002620
2621 return vma;
2622}
2623
2624struct i915_vma *
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02002625i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
2626 struct i915_address_space *vm)
Ben Widawsky6f65e292013-12-06 14:10:56 -08002627{
2628 struct i915_vma *vma;
2629
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02002630 vma = i915_gem_obj_to_vma(obj, vm);
Ben Widawsky6f65e292013-12-06 14:10:56 -08002631 if (!vma)
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02002632 vma = __i915_gem_vma_create(obj, vm,
2633 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL);
Ben Widawsky6f65e292013-12-06 14:10:56 -08002634
2635 return vma;
2636}
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00002637
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02002638struct i915_vma *
2639i915_gem_obj_lookup_or_create_ggtt_vma(struct drm_i915_gem_object *obj,
2640 const struct i915_ggtt_view *view)
2641{
2642 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
2643 struct i915_vma *vma;
2644
2645 if (WARN_ON(!view))
2646 return ERR_PTR(-EINVAL);
2647
2648 vma = i915_gem_obj_to_ggtt_view(obj, view);
2649
2650 if (IS_ERR(vma))
2651 return vma;
2652
2653 if (!vma)
2654 vma = __i915_gem_vma_create(obj, ggtt, view);
2655
2656 return vma;
2657
2658}
2659
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002660static void
2661rotate_pages(dma_addr_t *in, unsigned int width, unsigned int height,
2662 struct sg_table *st)
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00002663{
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002664 unsigned int column, row;
2665 unsigned int src_idx;
2666 struct scatterlist *sg = st->sgl;
2667
2668 st->nents = 0;
2669
2670 for (column = 0; column < width; column++) {
2671 src_idx = width * (height - 1) + column;
2672 for (row = 0; row < height; row++) {
2673 st->nents++;
2674 /* We don't need the pages, but need to initialize
2675 * the entries so the sg list can be happily traversed.
2676 * The only thing we need are DMA addresses.
2677 */
2678 sg_set_page(sg, NULL, PAGE_SIZE, 0);
2679 sg_dma_address(sg) = in[src_idx];
2680 sg_dma_len(sg) = PAGE_SIZE;
2681 sg = sg_next(sg);
2682 src_idx -= width;
2683 }
2684 }
2685}
2686
2687static struct sg_table *
2688intel_rotate_fb_obj_pages(struct i915_ggtt_view *ggtt_view,
2689 struct drm_i915_gem_object *obj)
2690{
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002691 struct intel_rotation_info *rot_info = &ggtt_view->rotation_info;
Tvrtko Ursulin84fe03f2015-06-23 14:26:46 +01002692 unsigned int size_pages = rot_info->size >> PAGE_SHIFT;
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002693 struct sg_page_iter sg_iter;
2694 unsigned long i;
2695 dma_addr_t *page_addr_list;
2696 struct sg_table *st;
Tvrtko Ursulin1d00dad2015-03-25 10:15:26 +00002697 int ret = -ENOMEM;
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002698
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002699 /* Allocate a temporary list of source pages for random access. */
Tvrtko Ursulin84fe03f2015-06-23 14:26:46 +01002700 page_addr_list = drm_malloc_ab(obj->base.size / PAGE_SIZE,
2701 sizeof(dma_addr_t));
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002702 if (!page_addr_list)
2703 return ERR_PTR(ret);
2704
2705 /* Allocate target SG list. */
2706 st = kmalloc(sizeof(*st), GFP_KERNEL);
2707 if (!st)
2708 goto err_st_alloc;
2709
Tvrtko Ursulin84fe03f2015-06-23 14:26:46 +01002710 ret = sg_alloc_table(st, size_pages, GFP_KERNEL);
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002711 if (ret)
2712 goto err_sg_alloc;
2713
2714 /* Populate source page list from the object. */
2715 i = 0;
2716 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
2717 page_addr_list[i] = sg_page_iter_dma_address(&sg_iter);
2718 i++;
2719 }
2720
2721 /* Rotate the pages. */
Tvrtko Ursulin84fe03f2015-06-23 14:26:46 +01002722 rotate_pages(page_addr_list,
2723 rot_info->width_pages, rot_info->height_pages,
2724 st);
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002725
2726 DRM_DEBUG_KMS(
Tvrtko Ursulin84fe03f2015-06-23 14:26:46 +01002727 "Created rotated page mapping for object size %zu (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages).\n",
Tvrtko Ursulinc9f8fd22015-06-24 09:55:20 +01002728 obj->base.size, rot_info->pitch, rot_info->height,
Tvrtko Ursulin84fe03f2015-06-23 14:26:46 +01002729 rot_info->pixel_format, rot_info->width_pages,
2730 rot_info->height_pages, size_pages);
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002731
2732 drm_free_large(page_addr_list);
2733
2734 return st;
2735
2736err_sg_alloc:
2737 kfree(st);
2738err_st_alloc:
2739 drm_free_large(page_addr_list);
2740
2741 DRM_DEBUG_KMS(
Tvrtko Ursulin84fe03f2015-06-23 14:26:46 +01002742 "Failed to create rotated mapping for object size %zu! (%d) (pitch=%u, height=%u, pixel_format=0x%x, %ux%u tiles, %u pages)\n",
Tvrtko Ursulinc9f8fd22015-06-24 09:55:20 +01002743 obj->base.size, ret, rot_info->pitch, rot_info->height,
Tvrtko Ursulin84fe03f2015-06-23 14:26:46 +01002744 rot_info->pixel_format, rot_info->width_pages,
2745 rot_info->height_pages, size_pages);
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002746 return ERR_PTR(ret);
2747}
2748
Joonas Lahtinen8bd7ef12015-05-06 14:35:38 +03002749static struct sg_table *
2750intel_partial_pages(const struct i915_ggtt_view *view,
2751 struct drm_i915_gem_object *obj)
2752{
2753 struct sg_table *st;
2754 struct scatterlist *sg;
2755 struct sg_page_iter obj_sg_iter;
2756 int ret = -ENOMEM;
2757
2758 st = kmalloc(sizeof(*st), GFP_KERNEL);
2759 if (!st)
2760 goto err_st_alloc;
2761
2762 ret = sg_alloc_table(st, view->params.partial.size, GFP_KERNEL);
2763 if (ret)
2764 goto err_sg_alloc;
2765
2766 sg = st->sgl;
2767 st->nents = 0;
2768 for_each_sg_page(obj->pages->sgl, &obj_sg_iter, obj->pages->nents,
2769 view->params.partial.offset)
2770 {
2771 if (st->nents >= view->params.partial.size)
2772 break;
2773
2774 sg_set_page(sg, NULL, PAGE_SIZE, 0);
2775 sg_dma_address(sg) = sg_page_iter_dma_address(&obj_sg_iter);
2776 sg_dma_len(sg) = PAGE_SIZE;
2777
2778 sg = sg_next(sg);
2779 st->nents++;
2780 }
2781
2782 return st;
2783
2784err_sg_alloc:
2785 kfree(st);
2786err_st_alloc:
2787 return ERR_PTR(ret);
2788}
2789
Daniel Vetter70b9f6f2015-04-14 17:35:27 +02002790static int
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002791i915_get_ggtt_vma_pages(struct i915_vma *vma)
2792{
2793 int ret = 0;
2794
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00002795 if (vma->ggtt_view.pages)
2796 return 0;
2797
2798 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
2799 vma->ggtt_view.pages = vma->obj->pages;
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002800 else if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
2801 vma->ggtt_view.pages =
2802 intel_rotate_fb_obj_pages(&vma->ggtt_view, vma->obj);
Joonas Lahtinen8bd7ef12015-05-06 14:35:38 +03002803 else if (vma->ggtt_view.type == I915_GGTT_VIEW_PARTIAL)
2804 vma->ggtt_view.pages =
2805 intel_partial_pages(&vma->ggtt_view, vma->obj);
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00002806 else
2807 WARN_ONCE(1, "GGTT view %u not implemented!\n",
2808 vma->ggtt_view.type);
2809
2810 if (!vma->ggtt_view.pages) {
Joonas Lahtinenec7adb62015-03-16 14:11:13 +02002811 DRM_ERROR("Failed to get pages for GGTT view type %u!\n",
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00002812 vma->ggtt_view.type);
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002813 ret = -EINVAL;
2814 } else if (IS_ERR(vma->ggtt_view.pages)) {
2815 ret = PTR_ERR(vma->ggtt_view.pages);
2816 vma->ggtt_view.pages = NULL;
2817 DRM_ERROR("Failed to get pages for VMA view type %u (%d)!\n",
2818 vma->ggtt_view.type, ret);
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00002819 }
2820
Tvrtko Ursulin50470bb2015-03-23 11:10:36 +00002821 return ret;
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00002822}
2823
2824/**
2825 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
2826 * @vma: VMA to map
2827 * @cache_level: mapping cache level
2828 * @flags: flags like global or local mapping
2829 *
2830 * DMA addresses are taken from the scatter-gather table of this object (or of
2831 * this VMA in case of non-default GGTT views) and PTE entries set up.
2832 * Note that DMA addresses are also the only part of the SG table we care about.
2833 */
2834int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
2835 u32 flags)
2836{
Mika Kuoppala75d04a32015-04-28 17:56:17 +03002837 int ret;
2838 u32 bind_flags;
Mika Kuoppala1d335d12015-04-10 15:54:58 +03002839
Mika Kuoppala75d04a32015-04-28 17:56:17 +03002840 if (WARN_ON(flags == 0))
2841 return -EINVAL;
Mika Kuoppala1d335d12015-04-10 15:54:58 +03002842
Mika Kuoppala75d04a32015-04-28 17:56:17 +03002843 bind_flags = 0;
Daniel Vetter08755462015-04-20 09:04:05 -07002844 if (flags & PIN_GLOBAL)
2845 bind_flags |= GLOBAL_BIND;
2846 if (flags & PIN_USER)
2847 bind_flags |= LOCAL_BIND;
2848
2849 if (flags & PIN_UPDATE)
2850 bind_flags |= vma->bound;
2851 else
2852 bind_flags &= ~vma->bound;
2853
Mika Kuoppala75d04a32015-04-28 17:56:17 +03002854 if (bind_flags == 0)
2855 return 0;
2856
2857 if (vma->bound == 0 && vma->vm->allocate_va_range) {
2858 trace_i915_va_alloc(vma->vm,
2859 vma->node.start,
2860 vma->node.size,
2861 VM_TO_TRACE_NAME(vma->vm));
2862
Mika Kuoppalab2dd4512015-06-25 18:35:15 +03002863 /* XXX: i915_vma_pin() will fix this +- hack */
2864 vma->pin_count++;
Mika Kuoppala75d04a32015-04-28 17:56:17 +03002865 ret = vma->vm->allocate_va_range(vma->vm,
2866 vma->node.start,
2867 vma->node.size);
Mika Kuoppalab2dd4512015-06-25 18:35:15 +03002868 vma->pin_count--;
Mika Kuoppala75d04a32015-04-28 17:56:17 +03002869 if (ret)
2870 return ret;
2871 }
2872
2873 ret = vma->vm->bind_vma(vma, cache_level, bind_flags);
Daniel Vetter70b9f6f2015-04-14 17:35:27 +02002874 if (ret)
2875 return ret;
Daniel Vetter08755462015-04-20 09:04:05 -07002876
2877 vma->bound |= bind_flags;
Tvrtko Ursulinfe14d5f2014-12-10 17:27:58 +00002878
2879 return 0;
2880}
Joonas Lahtinen91e67112015-05-06 14:33:58 +03002881
2882/**
2883 * i915_ggtt_view_size - Get the size of a GGTT view.
2884 * @obj: Object the view is of.
2885 * @view: The view in question.
2886 *
2887 * @return The size of the GGTT view in bytes.
2888 */
2889size_t
2890i915_ggtt_view_size(struct drm_i915_gem_object *obj,
2891 const struct i915_ggtt_view *view)
2892{
Tvrtko Ursulin9e759ff2015-06-23 12:57:43 +01002893 if (view->type == I915_GGTT_VIEW_NORMAL) {
Joonas Lahtinen91e67112015-05-06 14:33:58 +03002894 return obj->base.size;
Tvrtko Ursulin9e759ff2015-06-23 12:57:43 +01002895 } else if (view->type == I915_GGTT_VIEW_ROTATED) {
2896 return view->rotation_info.size;
Joonas Lahtinen8bd7ef12015-05-06 14:35:38 +03002897 } else if (view->type == I915_GGTT_VIEW_PARTIAL) {
2898 return view->params.partial.size << PAGE_SHIFT;
Joonas Lahtinen91e67112015-05-06 14:33:58 +03002899 } else {
2900 WARN_ONCE(1, "GGTT view %u not implemented!\n", view->type);
2901 return obj->base.size;
2902 }
2903}