blob: de9c945f72a7276dbf2a2cff0b04c6734535409f [file] [log] [blame]
Daniel Vetter76aaf222010-11-05 22:23:30 +01001/*
2 * Copyright © 2010 Daniel Vetter
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
David Howells760285e2012-10-02 18:01:07 +010025#include <drm/drmP.h>
26#include <drm/i915_drm.h>
Daniel Vetter76aaf222010-11-05 22:23:30 +010027#include "i915_drv.h"
28#include "i915_trace.h"
29#include "intel_drv.h"
30
Ben Widawsky6670a5a2013-06-27 16:30:04 -070031#define GEN6_PPGTT_PD_ENTRIES 512
32#define I915_PPGTT_PT_ENTRIES (PAGE_SIZE / sizeof(gen6_gtt_pte_t))
Ben Widawskyd31eb102013-11-02 21:07:17 -070033typedef uint64_t gen8_gtt_pte_t;
Ben Widawsky37aca442013-11-04 20:47:32 -080034typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;
Ben Widawsky6670a5a2013-06-27 16:30:04 -070035
Ben Widawsky26b1ff32012-11-04 09:21:31 -080036/* PPGTT stuff */
37#define GEN6_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0xff0))
Ben Widawsky0d8ff152013-07-04 11:02:03 -070038#define HSW_GTT_ADDR_ENCODE(addr) ((addr) | (((addr) >> 28) & 0x7f0))
Ben Widawsky26b1ff32012-11-04 09:21:31 -080039
40#define GEN6_PDE_VALID (1 << 0)
41/* gen6+ has bit 11-4 for physical addr bit 39-32 */
42#define GEN6_PDE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
43
44#define GEN6_PTE_VALID (1 << 0)
45#define GEN6_PTE_UNCACHED (1 << 1)
46#define HSW_PTE_UNCACHED (0)
47#define GEN6_PTE_CACHE_LLC (2 << 1)
Chris Wilson350ec882013-08-06 13:17:02 +010048#define GEN7_PTE_CACHE_L3_LLC (3 << 1)
Ben Widawsky26b1ff32012-11-04 09:21:31 -080049#define GEN6_PTE_ADDR_ENCODE(addr) GEN6_GTT_ADDR_ENCODE(addr)
Ben Widawsky0d8ff152013-07-04 11:02:03 -070050#define HSW_PTE_ADDR_ENCODE(addr) HSW_GTT_ADDR_ENCODE(addr)
51
52/* Cacheability Control is a 4-bit value. The low three bits are stored in *
53 * bits 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE.
54 */
55#define HSW_CACHEABILITY_CONTROL(bits) ((((bits) & 0x7) << 1) | \
56 (((bits) & 0x8) << (11 - 3)))
Ben Widawsky87a6b682013-08-04 23:47:29 -070057#define HSW_WB_LLC_AGE3 HSW_CACHEABILITY_CONTROL(0x2)
Ben Widawsky0d8ff152013-07-04 11:02:03 -070058#define HSW_WB_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x3)
Ben Widawsky4d15c142013-07-04 11:02:06 -070059#define HSW_WB_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0xb)
Chris Wilson651d7942013-08-08 14:41:10 +010060#define HSW_WT_ELLC_LLC_AGE0 HSW_CACHEABILITY_CONTROL(0x6)
Ben Widawsky26b1ff32012-11-04 09:21:31 -080061
Ben Widawsky459108b2013-11-02 21:07:23 -070062#define GEN8_PTES_PER_PAGE (PAGE_SIZE / sizeof(gen8_gtt_pte_t))
Ben Widawsky37aca442013-11-04 20:47:32 -080063#define GEN8_PDES_PER_PAGE (PAGE_SIZE / sizeof(gen8_ppgtt_pde_t))
64#define GEN8_LEGACY_PDPS 4
65
Ben Widawskyfbe5d362013-11-04 19:56:49 -080066#define PPAT_UNCACHED_INDEX (_PAGE_PWT | _PAGE_PCD)
67#define PPAT_CACHED_PDE_INDEX 0 /* WB LLC */
68#define PPAT_CACHED_INDEX _PAGE_PAT /* WB LLCeLLC */
69#define PPAT_DISPLAY_ELLC_INDEX _PAGE_PCD /* WT eLLC */
70
Ben Widawsky94ec8f62013-11-02 21:07:18 -070071static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
72 enum i915_cache_level level,
73 bool valid)
74{
75 gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
76 pte |= addr;
Ben Widawskyfbe5d362013-11-04 19:56:49 -080077 if (level != I915_CACHE_NONE)
78 pte |= PPAT_CACHED_INDEX;
79 else
80 pte |= PPAT_UNCACHED_INDEX;
Ben Widawsky94ec8f62013-11-02 21:07:18 -070081 return pte;
82}
83
Ben Widawskyb1fe6672013-11-04 21:20:14 -080084static inline gen8_ppgtt_pde_t gen8_pde_encode(struct drm_device *dev,
85 dma_addr_t addr,
86 enum i915_cache_level level)
87{
88 gen8_ppgtt_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
89 pde |= addr;
90 if (level != I915_CACHE_NONE)
91 pde |= PPAT_CACHED_PDE_INDEX;
92 else
93 pde |= PPAT_UNCACHED_INDEX;
94 return pde;
95}
96
Chris Wilson350ec882013-08-06 13:17:02 +010097static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
Ben Widawskyb35b3802013-10-16 09:18:21 -070098 enum i915_cache_level level,
99 bool valid)
Ben Widawsky54d12522012-09-24 16:44:32 -0700100{
Ben Widawskyb35b3802013-10-16 09:18:21 -0700101 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Ben Widawsky54d12522012-09-24 16:44:32 -0700102 pte |= GEN6_PTE_ADDR_ENCODE(addr);
Ben Widawskye7210c32012-10-19 09:33:22 -0700103
104 switch (level) {
Chris Wilson350ec882013-08-06 13:17:02 +0100105 case I915_CACHE_L3_LLC:
106 case I915_CACHE_LLC:
107 pte |= GEN6_PTE_CACHE_LLC;
108 break;
109 case I915_CACHE_NONE:
110 pte |= GEN6_PTE_UNCACHED;
111 break;
112 default:
113 WARN_ON(1);
114 }
115
116 return pte;
117}
118
119static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
Ben Widawskyb35b3802013-10-16 09:18:21 -0700120 enum i915_cache_level level,
121 bool valid)
Chris Wilson350ec882013-08-06 13:17:02 +0100122{
Ben Widawskyb35b3802013-10-16 09:18:21 -0700123 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Chris Wilson350ec882013-08-06 13:17:02 +0100124 pte |= GEN6_PTE_ADDR_ENCODE(addr);
125
126 switch (level) {
127 case I915_CACHE_L3_LLC:
128 pte |= GEN7_PTE_CACHE_L3_LLC;
Ben Widawskye7210c32012-10-19 09:33:22 -0700129 break;
130 case I915_CACHE_LLC:
131 pte |= GEN6_PTE_CACHE_LLC;
132 break;
133 case I915_CACHE_NONE:
Kenneth Graunke91197082013-04-22 00:53:51 -0700134 pte |= GEN6_PTE_UNCACHED;
Ben Widawskye7210c32012-10-19 09:33:22 -0700135 break;
136 default:
Chris Wilson350ec882013-08-06 13:17:02 +0100137 WARN_ON(1);
Ben Widawskye7210c32012-10-19 09:33:22 -0700138 }
139
Ben Widawsky54d12522012-09-24 16:44:32 -0700140 return pte;
141}
142
Kenneth Graunke93c34e72013-04-22 00:53:50 -0700143#define BYT_PTE_WRITEABLE (1 << 1)
144#define BYT_PTE_SNOOPED_BY_CPU_CACHES (1 << 2)
145
Ben Widawsky80a74f72013-06-27 16:30:19 -0700146static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
Ben Widawskyb35b3802013-10-16 09:18:21 -0700147 enum i915_cache_level level,
148 bool valid)
Kenneth Graunke93c34e72013-04-22 00:53:50 -0700149{
Ben Widawskyb35b3802013-10-16 09:18:21 -0700150 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Kenneth Graunke93c34e72013-04-22 00:53:50 -0700151 pte |= GEN6_PTE_ADDR_ENCODE(addr);
152
153 /* Mark the page as writeable. Other platforms don't have a
154 * setting for read-only/writable, so this matches that behavior.
155 */
156 pte |= BYT_PTE_WRITEABLE;
157
158 if (level != I915_CACHE_NONE)
159 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
160
161 return pte;
162}
163
Ben Widawsky80a74f72013-06-27 16:30:19 -0700164static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
Ben Widawskyb35b3802013-10-16 09:18:21 -0700165 enum i915_cache_level level,
166 bool valid)
Kenneth Graunke91197082013-04-22 00:53:51 -0700167{
Ben Widawskyb35b3802013-10-16 09:18:21 -0700168 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Ben Widawsky0d8ff152013-07-04 11:02:03 -0700169 pte |= HSW_PTE_ADDR_ENCODE(addr);
Kenneth Graunke91197082013-04-22 00:53:51 -0700170
171 if (level != I915_CACHE_NONE)
Ben Widawsky87a6b682013-08-04 23:47:29 -0700172 pte |= HSW_WB_LLC_AGE3;
Kenneth Graunke91197082013-04-22 00:53:51 -0700173
174 return pte;
175}
176
Ben Widawsky4d15c142013-07-04 11:02:06 -0700177static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
Ben Widawskyb35b3802013-10-16 09:18:21 -0700178 enum i915_cache_level level,
179 bool valid)
Ben Widawsky4d15c142013-07-04 11:02:06 -0700180{
Ben Widawskyb35b3802013-10-16 09:18:21 -0700181 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Ben Widawsky4d15c142013-07-04 11:02:06 -0700182 pte |= HSW_PTE_ADDR_ENCODE(addr);
183
Chris Wilson651d7942013-08-08 14:41:10 +0100184 switch (level) {
185 case I915_CACHE_NONE:
186 break;
187 case I915_CACHE_WT:
188 pte |= HSW_WT_ELLC_LLC_AGE0;
189 break;
190 default:
Ben Widawsky4d15c142013-07-04 11:02:06 -0700191 pte |= HSW_WB_ELLC_LLC_AGE0;
Chris Wilson651d7942013-08-08 14:41:10 +0100192 break;
193 }
Ben Widawsky4d15c142013-07-04 11:02:06 -0700194
195 return pte;
196}
197
Ben Widawsky94e409c2013-11-04 22:29:36 -0800198/* Broadwell Page Directory Pointer Descriptors */
199static int gen8_write_pdp(struct intel_ring_buffer *ring, unsigned entry,
200 uint64_t val)
201{
202 int ret;
203
204 BUG_ON(entry >= 4);
205
206 ret = intel_ring_begin(ring, 6);
207 if (ret)
208 return ret;
209
210 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
211 intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
212 intel_ring_emit(ring, (u32)(val >> 32));
213 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
214 intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
215 intel_ring_emit(ring, (u32)(val));
216 intel_ring_advance(ring);
217
218 return 0;
219}
220
221static int gen8_ppgtt_enable(struct drm_device *dev)
222{
223 struct drm_i915_private *dev_priv = dev->dev_private;
224 struct intel_ring_buffer *ring;
225 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
226 int i, j, ret;
227
228 /* bit of a hack to find the actual last used pd */
229 int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
230
231 for_each_ring(ring, dev_priv, j) {
232 I915_WRITE(RING_MODE_GEN7(ring),
233 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
234 }
235
236 for (i = used_pd - 1; i >= 0; i--) {
237 dma_addr_t addr = ppgtt->pd_dma_addr[i];
238 for_each_ring(ring, dev_priv, j) {
239 ret = gen8_write_pdp(ring, i, addr);
240 if (ret)
Ben Widawskyd595bd42013-11-25 09:54:32 -0800241 goto err_out;
Ben Widawsky94e409c2013-11-04 22:29:36 -0800242 }
243 }
244 return 0;
Ben Widawskyd595bd42013-11-25 09:54:32 -0800245
246err_out:
247 for_each_ring(ring, dev_priv, j)
248 I915_WRITE(RING_MODE_GEN7(ring),
249 _MASKED_BIT_DISABLE(GFX_PPGTT_ENABLE));
250 return ret;
Ben Widawsky94e409c2013-11-04 22:29:36 -0800251}
252
Ben Widawsky459108b2013-11-02 21:07:23 -0700253static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
254 unsigned first_entry,
255 unsigned num_entries,
256 bool use_scratch)
257{
258 struct i915_hw_ppgtt *ppgtt =
259 container_of(vm, struct i915_hw_ppgtt, base);
260 gen8_gtt_pte_t *pt_vaddr, scratch_pte;
261 unsigned act_pt = first_entry / GEN8_PTES_PER_PAGE;
262 unsigned first_pte = first_entry % GEN8_PTES_PER_PAGE;
263 unsigned last_pte, i;
264
265 scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
266 I915_CACHE_LLC, use_scratch);
267
268 while (num_entries) {
269 struct page *page_table = &ppgtt->gen8_pt_pages[act_pt];
270
271 last_pte = first_pte + num_entries;
272 if (last_pte > GEN8_PTES_PER_PAGE)
273 last_pte = GEN8_PTES_PER_PAGE;
274
275 pt_vaddr = kmap_atomic(page_table);
276
277 for (i = first_pte; i < last_pte; i++)
278 pt_vaddr[i] = scratch_pte;
279
280 kunmap_atomic(pt_vaddr);
281
282 num_entries -= last_pte - first_pte;
283 first_pte = 0;
284 act_pt++;
285 }
286}
287
Ben Widawsky9df15b42013-11-02 21:07:24 -0700288static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
289 struct sg_table *pages,
290 unsigned first_entry,
291 enum i915_cache_level cache_level)
292{
293 struct i915_hw_ppgtt *ppgtt =
294 container_of(vm, struct i915_hw_ppgtt, base);
295 gen8_gtt_pte_t *pt_vaddr;
296 unsigned act_pt = first_entry / GEN8_PTES_PER_PAGE;
297 unsigned act_pte = first_entry % GEN8_PTES_PER_PAGE;
298 struct sg_page_iter sg_iter;
299
300 pt_vaddr = kmap_atomic(&ppgtt->gen8_pt_pages[act_pt]);
301 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
302 dma_addr_t page_addr;
303
304 page_addr = sg_dma_address(sg_iter.sg) +
305 (sg_iter.sg_pgoffset << PAGE_SHIFT);
306 pt_vaddr[act_pte] = gen8_pte_encode(page_addr, cache_level,
307 true);
308 if (++act_pte == GEN8_PTES_PER_PAGE) {
309 kunmap_atomic(pt_vaddr);
310 act_pt++;
311 pt_vaddr = kmap_atomic(&ppgtt->gen8_pt_pages[act_pt]);
312 act_pte = 0;
313
314 }
315 }
316 kunmap_atomic(pt_vaddr);
317}
318
Ben Widawsky37aca442013-11-04 20:47:32 -0800319static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
320{
321 struct i915_hw_ppgtt *ppgtt =
322 container_of(vm, struct i915_hw_ppgtt, base);
323 int i, j;
324
Ben Widawsky686e1f6f2013-11-25 09:54:34 -0800325 drm_mm_takedown(&vm->mm);
326
Ben Widawsky37aca442013-11-04 20:47:32 -0800327 for (i = 0; i < ppgtt->num_pd_pages ; i++) {
328 if (ppgtt->pd_dma_addr[i]) {
329 pci_unmap_page(ppgtt->base.dev->pdev,
330 ppgtt->pd_dma_addr[i],
331 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
332
333 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
334 dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
335 if (addr)
336 pci_unmap_page(ppgtt->base.dev->pdev,
337 addr,
338 PAGE_SIZE,
339 PCI_DMA_BIDIRECTIONAL);
340
341 }
342 }
343 kfree(ppgtt->gen8_pt_dma_addr[i]);
344 }
345
Ben Widawsky230f9552013-11-07 21:40:48 -0800346 __free_pages(ppgtt->gen8_pt_pages, get_order(ppgtt->num_pt_pages << PAGE_SHIFT));
347 __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
Ben Widawsky37aca442013-11-04 20:47:32 -0800348}
349
350/**
351 * GEN8 legacy ppgtt programming is accomplished through 4 PDP registers with a
352 * net effect resembling a 2-level page table in normal x86 terms. Each PDP
353 * represents 1GB of memory
354 * 4 * 512 * 512 * 4096 = 4GB legacy 32b address space.
355 *
356 * TODO: Do something with the size parameter
357 **/
358static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
359{
360 struct page *pt_pages;
361 int i, j, ret = -ENOMEM;
362 const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
363 const int num_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
364
365 if (size % (1<<30))
366 DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size);
367
368 /* FIXME: split allocation into smaller pieces. For now we only ever do
369 * this once, but with full PPGTT, the multiple contiguous allocations
370 * will be bad.
371 */
372 ppgtt->pd_pages = alloc_pages(GFP_KERNEL, get_order(max_pdp << PAGE_SHIFT));
373 if (!ppgtt->pd_pages)
374 return -ENOMEM;
375
376 pt_pages = alloc_pages(GFP_KERNEL, get_order(num_pt_pages << PAGE_SHIFT));
377 if (!pt_pages) {
378 __free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT));
379 return -ENOMEM;
380 }
381
382 ppgtt->gen8_pt_pages = pt_pages;
383 ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT);
384 ppgtt->num_pt_pages = 1 << get_order(num_pt_pages << PAGE_SHIFT);
385 ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE;
Ben Widawsky94e409c2013-11-04 22:29:36 -0800386 ppgtt->enable = gen8_ppgtt_enable;
Ben Widawsky459108b2013-11-02 21:07:23 -0700387 ppgtt->base.clear_range = gen8_ppgtt_clear_range;
Ben Widawsky9df15b42013-11-02 21:07:24 -0700388 ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
Ben Widawsky37aca442013-11-04 20:47:32 -0800389 ppgtt->base.cleanup = gen8_ppgtt_cleanup;
Ben Widawsky686e1f6f2013-11-25 09:54:34 -0800390 ppgtt->base.start = 0;
391 ppgtt->base.total = ppgtt->num_pt_pages * GEN8_PTES_PER_PAGE * PAGE_SIZE;
Ben Widawsky37aca442013-11-04 20:47:32 -0800392
393 BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS);
394
395 /*
396 * - Create a mapping for the page directories.
397 * - For each page directory:
398 * allocate space for page table mappings.
399 * map each page table
400 */
401 for (i = 0; i < max_pdp; i++) {
402 dma_addr_t temp;
403 temp = pci_map_page(ppgtt->base.dev->pdev,
404 &ppgtt->pd_pages[i], 0,
405 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
406 if (pci_dma_mapping_error(ppgtt->base.dev->pdev, temp))
407 goto err_out;
408
409 ppgtt->pd_dma_addr[i] = temp;
410
411 ppgtt->gen8_pt_dma_addr[i] = kmalloc(sizeof(dma_addr_t) * GEN8_PDES_PER_PAGE, GFP_KERNEL);
412 if (!ppgtt->gen8_pt_dma_addr[i])
413 goto err_out;
414
415 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
416 struct page *p = &pt_pages[i * GEN8_PDES_PER_PAGE + j];
417 temp = pci_map_page(ppgtt->base.dev->pdev,
418 p, 0, PAGE_SIZE,
419 PCI_DMA_BIDIRECTIONAL);
420
421 if (pci_dma_mapping_error(ppgtt->base.dev->pdev, temp))
422 goto err_out;
423
424 ppgtt->gen8_pt_dma_addr[i][j] = temp;
425 }
426 }
427
Ben Widawskyb1fe6672013-11-04 21:20:14 -0800428 /* For now, the PPGTT helper functions all require that the PDEs are
429 * plugged in correctly. So we do that now/here. For aliasing PPGTT, we
430 * will never need to touch the PDEs again */
431 for (i = 0; i < max_pdp; i++) {
432 gen8_ppgtt_pde_t *pd_vaddr;
433 pd_vaddr = kmap_atomic(&ppgtt->pd_pages[i]);
434 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
435 dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
436 pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
437 I915_CACHE_LLC);
438 }
439 kunmap_atomic(pd_vaddr);
440 }
441
Ben Widawsky459108b2013-11-02 21:07:23 -0700442 ppgtt->base.clear_range(&ppgtt->base, 0,
443 ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE,
444 true);
445
Ben Widawsky37aca442013-11-04 20:47:32 -0800446 DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n",
447 ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
448 DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",
449 ppgtt->num_pt_pages,
450 (ppgtt->num_pt_pages - num_pt_pages) +
451 size % (1<<30));
Ben Widawsky28cf5412013-11-02 21:07:26 -0700452 return 0;
Ben Widawsky37aca442013-11-04 20:47:32 -0800453
454err_out:
455 ppgtt->base.cleanup(&ppgtt->base);
456 return ret;
457}
458
Ben Widawsky3e302542013-04-23 23:15:32 -0700459static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
Ben Widawsky61973492013-04-08 18:43:54 -0700460{
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700461 struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
Ben Widawsky61973492013-04-08 18:43:54 -0700462 gen6_gtt_pte_t __iomem *pd_addr;
463 uint32_t pd_entry;
464 int i;
465
Ben Widawsky0a732872013-04-23 23:15:30 -0700466 WARN_ON(ppgtt->pd_offset & 0x3f);
Ben Widawsky61973492013-04-08 18:43:54 -0700467 pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
468 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
469 for (i = 0; i < ppgtt->num_pd_entries; i++) {
470 dma_addr_t pt_addr;
471
472 pt_addr = ppgtt->pt_dma_addr[i];
473 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
474 pd_entry |= GEN6_PDE_VALID;
475
476 writel(pd_entry, pd_addr + i);
477 }
478 readl(pd_addr);
Ben Widawsky3e302542013-04-23 23:15:32 -0700479}
480
481static int gen6_ppgtt_enable(struct drm_device *dev)
482{
483 drm_i915_private_t *dev_priv = dev->dev_private;
484 uint32_t pd_offset;
485 struct intel_ring_buffer *ring;
486 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
487 int i;
488
489 BUG_ON(ppgtt->pd_offset & 0x3f);
490
491 gen6_write_pdes(ppgtt);
Ben Widawsky61973492013-04-08 18:43:54 -0700492
493 pd_offset = ppgtt->pd_offset;
494 pd_offset /= 64; /* in cachelines, */
495 pd_offset <<= 16;
496
497 if (INTEL_INFO(dev)->gen == 6) {
498 uint32_t ecochk, gab_ctl, ecobits;
499
500 ecobits = I915_READ(GAC_ECO_BITS);
Ville Syrjälä3b9d7882013-04-04 15:13:40 +0300501 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
502 ECOBITS_PPGTT_CACHE64B);
Ben Widawsky61973492013-04-08 18:43:54 -0700503
504 gab_ctl = I915_READ(GAB_CTL);
505 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
506
507 ecochk = I915_READ(GAM_ECOCHK);
508 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
509 ECOCHK_PPGTT_CACHE64B);
510 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
511 } else if (INTEL_INFO(dev)->gen >= 7) {
Ville Syrjäläa6f429a2013-04-04 15:13:42 +0300512 uint32_t ecochk, ecobits;
Ville Syrjäläa65c2fc2013-04-04 15:13:41 +0300513
514 ecobits = I915_READ(GAC_ECO_BITS);
515 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
516
Ville Syrjäläa6f429a2013-04-04 15:13:42 +0300517 ecochk = I915_READ(GAM_ECOCHK);
518 if (IS_HASWELL(dev)) {
519 ecochk |= ECOCHK_PPGTT_WB_HSW;
520 } else {
521 ecochk |= ECOCHK_PPGTT_LLC_IVB;
522 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
523 }
524 I915_WRITE(GAM_ECOCHK, ecochk);
Ben Widawsky61973492013-04-08 18:43:54 -0700525 /* GFX_MODE is per-ring on gen7+ */
526 }
527
528 for_each_ring(ring, dev_priv, i) {
529 if (INTEL_INFO(dev)->gen >= 7)
530 I915_WRITE(RING_MODE_GEN7(ring),
531 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
532
533 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
534 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
535 }
Ben Widawskyb7c36d22013-04-08 18:43:56 -0700536 return 0;
Ben Widawsky61973492013-04-08 18:43:54 -0700537}
538
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100539/* PPGTT support for Sandybdrige/Gen6 and later */
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700540static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100541 unsigned first_entry,
Ben Widawsky828c7902013-10-16 09:21:30 -0700542 unsigned num_entries,
543 bool use_scratch)
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100544{
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700545 struct i915_hw_ppgtt *ppgtt =
546 container_of(vm, struct i915_hw_ppgtt, base);
Ben Widawskye7c2b582013-04-08 18:43:48 -0700547 gen6_gtt_pte_t *pt_vaddr, scratch_pte;
Daniel Vettera15326a2013-03-19 23:48:39 +0100548 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
Daniel Vetter7bddb012012-02-09 17:15:47 +0100549 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
550 unsigned last_pte, i;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100551
Ben Widawskyb35b3802013-10-16 09:18:21 -0700552 scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true);
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100553
Daniel Vetter7bddb012012-02-09 17:15:47 +0100554 while (num_entries) {
555 last_pte = first_pte + num_entries;
556 if (last_pte > I915_PPGTT_PT_ENTRIES)
557 last_pte = I915_PPGTT_PT_ENTRIES;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100558
Daniel Vettera15326a2013-03-19 23:48:39 +0100559 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
Daniel Vetter7bddb012012-02-09 17:15:47 +0100560
561 for (i = first_pte; i < last_pte; i++)
562 pt_vaddr[i] = scratch_pte;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100563
564 kunmap_atomic(pt_vaddr);
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100565
Daniel Vetter7bddb012012-02-09 17:15:47 +0100566 num_entries -= last_pte - first_pte;
567 first_pte = 0;
Daniel Vettera15326a2013-03-19 23:48:39 +0100568 act_pt++;
Daniel Vetter7bddb012012-02-09 17:15:47 +0100569 }
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100570}
571
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700572static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
Daniel Vetterdef886c2013-01-24 14:44:56 -0800573 struct sg_table *pages,
574 unsigned first_entry,
575 enum i915_cache_level cache_level)
576{
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700577 struct i915_hw_ppgtt *ppgtt =
578 container_of(vm, struct i915_hw_ppgtt, base);
Ben Widawskye7c2b582013-04-08 18:43:48 -0700579 gen6_gtt_pte_t *pt_vaddr;
Daniel Vettera15326a2013-03-19 23:48:39 +0100580 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
Imre Deak6e995e22013-02-18 19:28:04 +0200581 unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
582 struct sg_page_iter sg_iter;
Daniel Vetterdef886c2013-01-24 14:44:56 -0800583
Chris Wilsoncc797142013-12-31 15:50:30 +0000584 pt_vaddr = NULL;
Imre Deak6e995e22013-02-18 19:28:04 +0200585 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
Chris Wilsoncc797142013-12-31 15:50:30 +0000586 if (pt_vaddr == NULL)
587 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
Daniel Vetterdef886c2013-01-24 14:44:56 -0800588
Chris Wilsoncc797142013-12-31 15:50:30 +0000589 pt_vaddr[act_pte] =
590 vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
591 cache_level, true);
Imre Deak6e995e22013-02-18 19:28:04 +0200592 if (++act_pte == I915_PPGTT_PT_ENTRIES) {
593 kunmap_atomic(pt_vaddr);
Chris Wilsoncc797142013-12-31 15:50:30 +0000594 pt_vaddr = NULL;
Daniel Vettera15326a2013-03-19 23:48:39 +0100595 act_pt++;
Imre Deak6e995e22013-02-18 19:28:04 +0200596 act_pte = 0;
Daniel Vetterdef886c2013-01-24 14:44:56 -0800597 }
Daniel Vetterdef886c2013-01-24 14:44:56 -0800598 }
Chris Wilsoncc797142013-12-31 15:50:30 +0000599 if (pt_vaddr)
600 kunmap_atomic(pt_vaddr);
Daniel Vetterdef886c2013-01-24 14:44:56 -0800601}
602
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700603static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100604{
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700605 struct i915_hw_ppgtt *ppgtt =
606 container_of(vm, struct i915_hw_ppgtt, base);
Daniel Vetter3440d262013-01-24 13:49:56 -0800607 int i;
608
Ben Widawsky93bd8642013-07-16 16:50:06 -0700609 drm_mm_takedown(&ppgtt->base.mm);
610
Daniel Vetter3440d262013-01-24 13:49:56 -0800611 if (ppgtt->pt_dma_addr) {
612 for (i = 0; i < ppgtt->num_pd_entries; i++)
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700613 pci_unmap_page(ppgtt->base.dev->pdev,
Daniel Vetter3440d262013-01-24 13:49:56 -0800614 ppgtt->pt_dma_addr[i],
615 4096, PCI_DMA_BIDIRECTIONAL);
616 }
617
618 kfree(ppgtt->pt_dma_addr);
619 for (i = 0; i < ppgtt->num_pd_entries; i++)
620 __free_page(ppgtt->pt_pages[i]);
621 kfree(ppgtt->pt_pages);
622 kfree(ppgtt);
623}
624
625static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
626{
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700627 struct drm_device *dev = ppgtt->base.dev;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100628 struct drm_i915_private *dev_priv = dev->dev_private;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100629 unsigned first_pd_entry_in_global_pt;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100630 int i;
631 int ret = -ENOMEM;
632
633 /* ppgtt PDEs reside in the global gtt pagetable, which has 512*1024
634 * entries. For aliasing ppgtt support we just steal them at the end for
635 * now. */
Daniel Vettere1b73cb2013-05-21 09:52:16 +0200636 first_pd_entry_in_global_pt = gtt_total_entries(dev_priv->gtt);
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100637
Chris Wilson08c45262013-07-30 19:04:37 +0100638 ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
Ben Widawsky6670a5a2013-06-27 16:30:04 -0700639 ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
Ben Widawsky61973492013-04-08 18:43:54 -0700640 ppgtt->enable = gen6_ppgtt_enable;
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700641 ppgtt->base.clear_range = gen6_ppgtt_clear_range;
642 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
643 ppgtt->base.cleanup = gen6_ppgtt_cleanup;
644 ppgtt->base.scratch = dev_priv->gtt.base.scratch;
Ben Widawsky686e1f6f2013-11-25 09:54:34 -0800645 ppgtt->base.start = 0;
646 ppgtt->base.total = GEN6_PPGTT_PD_ENTRIES * I915_PPGTT_PT_ENTRIES * PAGE_SIZE;
Daniel Vettera1e22652013-09-21 00:35:38 +0200647 ppgtt->pt_pages = kcalloc(ppgtt->num_pd_entries, sizeof(struct page *),
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100648 GFP_KERNEL);
649 if (!ppgtt->pt_pages)
Daniel Vetter3440d262013-01-24 13:49:56 -0800650 return -ENOMEM;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100651
652 for (i = 0; i < ppgtt->num_pd_entries; i++) {
653 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
654 if (!ppgtt->pt_pages[i])
655 goto err_pt_alloc;
656 }
657
Daniel Vettera1e22652013-09-21 00:35:38 +0200658 ppgtt->pt_dma_addr = kcalloc(ppgtt->num_pd_entries, sizeof(dma_addr_t),
Ben Widawsky8d2e6302013-01-18 12:30:33 -0800659 GFP_KERNEL);
660 if (!ppgtt->pt_dma_addr)
661 goto err_pt_alloc;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100662
Ben Widawsky8d2e6302013-01-18 12:30:33 -0800663 for (i = 0; i < ppgtt->num_pd_entries; i++) {
664 dma_addr_t pt_addr;
Daniel Vetter211c5682012-04-10 17:29:17 +0200665
Ben Widawsky8d2e6302013-01-18 12:30:33 -0800666 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
667 PCI_DMA_BIDIRECTIONAL);
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100668
Ben Widawsky8d2e6302013-01-18 12:30:33 -0800669 if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
670 ret = -EIO;
671 goto err_pd_pin;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100672
Daniel Vetter211c5682012-04-10 17:29:17 +0200673 }
Ben Widawsky8d2e6302013-01-18 12:30:33 -0800674 ppgtt->pt_dma_addr[i] = pt_addr;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100675 }
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100676
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700677 ppgtt->base.clear_range(&ppgtt->base, 0,
Ben Widawsky828c7902013-10-16 09:21:30 -0700678 ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES, true);
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100679
Ben Widawskye7c2b582013-04-08 18:43:48 -0700680 ppgtt->pd_offset = first_pd_entry_in_global_pt * sizeof(gen6_gtt_pte_t);
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100681
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100682 return 0;
683
684err_pd_pin:
685 if (ppgtt->pt_dma_addr) {
686 for (i--; i >= 0; i--)
687 pci_unmap_page(dev->pdev, ppgtt->pt_dma_addr[i],
688 4096, PCI_DMA_BIDIRECTIONAL);
689 }
690err_pt_alloc:
691 kfree(ppgtt->pt_dma_addr);
692 for (i = 0; i < ppgtt->num_pd_entries; i++) {
693 if (ppgtt->pt_pages[i])
694 __free_page(ppgtt->pt_pages[i]);
695 }
696 kfree(ppgtt->pt_pages);
Daniel Vetter3440d262013-01-24 13:49:56 -0800697
698 return ret;
699}
700
701static int i915_gem_init_aliasing_ppgtt(struct drm_device *dev)
702{
703 struct drm_i915_private *dev_priv = dev->dev_private;
704 struct i915_hw_ppgtt *ppgtt;
705 int ret;
706
707 ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
708 if (!ppgtt)
709 return -ENOMEM;
710
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700711 ppgtt->base.dev = dev;
Daniel Vetter3440d262013-01-24 13:49:56 -0800712
Ben Widawsky3ed124b2013-04-08 18:43:53 -0700713 if (INTEL_INFO(dev)->gen < 8)
714 ret = gen6_ppgtt_init(ppgtt);
Daniel Vetter8fe6bd22013-11-02 21:07:01 -0700715 else if (IS_GEN8(dev))
Ben Widawsky37aca442013-11-04 20:47:32 -0800716 ret = gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
Ben Widawsky3ed124b2013-04-08 18:43:53 -0700717 else
718 BUG();
719
Daniel Vetter3440d262013-01-24 13:49:56 -0800720 if (ret)
721 kfree(ppgtt);
Ben Widawsky93bd8642013-07-16 16:50:06 -0700722 else {
Daniel Vetter3440d262013-01-24 13:49:56 -0800723 dev_priv->mm.aliasing_ppgtt = ppgtt;
Ben Widawsky93bd8642013-07-16 16:50:06 -0700724 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
725 ppgtt->base.total);
726 }
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100727
728 return ret;
729}
730
731void i915_gem_cleanup_aliasing_ppgtt(struct drm_device *dev)
732{
733 struct drm_i915_private *dev_priv = dev->dev_private;
734 struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100735
736 if (!ppgtt)
737 return;
738
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700739 ppgtt->base.cleanup(&ppgtt->base);
Ben Widawsky5963cf02013-04-08 18:43:55 -0700740 dev_priv->mm.aliasing_ppgtt = NULL;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100741}
742
Daniel Vetter7bddb012012-02-09 17:15:47 +0100743void i915_ppgtt_bind_object(struct i915_hw_ppgtt *ppgtt,
744 struct drm_i915_gem_object *obj,
745 enum i915_cache_level cache_level)
746{
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700747 ppgtt->base.insert_entries(&ppgtt->base, obj->pages,
748 i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
749 cache_level);
Daniel Vetter7bddb012012-02-09 17:15:47 +0100750}
751
752void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
753 struct drm_i915_gem_object *obj)
754{
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700755 ppgtt->base.clear_range(&ppgtt->base,
756 i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT,
Ben Widawsky828c7902013-10-16 09:21:30 -0700757 obj->base.size >> PAGE_SHIFT,
758 true);
Daniel Vetter7bddb012012-02-09 17:15:47 +0100759}
760
Ben Widawskya81cc002013-01-18 12:30:31 -0800761extern int intel_iommu_gfx_mapped;
762/* Certain Gen5 chipsets require require idling the GPU before
763 * unmapping anything from the GTT when VT-d is enabled.
764 */
765static inline bool needs_idle_maps(struct drm_device *dev)
766{
767#ifdef CONFIG_INTEL_IOMMU
768 /* Query intel_iommu to see if we need the workaround. Presumably that
769 * was loaded first.
770 */
771 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
772 return true;
773#endif
774 return false;
775}
776
Ben Widawsky5c042282011-10-17 15:51:55 -0700777static bool do_idling(struct drm_i915_private *dev_priv)
778{
779 bool ret = dev_priv->mm.interruptible;
780
Ben Widawskya81cc002013-01-18 12:30:31 -0800781 if (unlikely(dev_priv->gtt.do_idle_maps)) {
Ben Widawsky5c042282011-10-17 15:51:55 -0700782 dev_priv->mm.interruptible = false;
Ben Widawskyb2da9fe2012-04-26 16:02:58 -0700783 if (i915_gpu_idle(dev_priv->dev)) {
Ben Widawsky5c042282011-10-17 15:51:55 -0700784 DRM_ERROR("Couldn't idle GPU\n");
785 /* Wait a bit, in hopes it avoids the hang */
786 udelay(10);
787 }
788 }
789
790 return ret;
791}
792
793static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
794{
Ben Widawskya81cc002013-01-18 12:30:31 -0800795 if (unlikely(dev_priv->gtt.do_idle_maps))
Ben Widawsky5c042282011-10-17 15:51:55 -0700796 dev_priv->mm.interruptible = interruptible;
797}
798
Ben Widawsky828c7902013-10-16 09:21:30 -0700799void i915_check_and_clear_faults(struct drm_device *dev)
800{
801 struct drm_i915_private *dev_priv = dev->dev_private;
802 struct intel_ring_buffer *ring;
803 int i;
804
805 if (INTEL_INFO(dev)->gen < 6)
806 return;
807
808 for_each_ring(ring, dev_priv, i) {
809 u32 fault_reg;
810 fault_reg = I915_READ(RING_FAULT_REG(ring));
811 if (fault_reg & RING_FAULT_VALID) {
812 DRM_DEBUG_DRIVER("Unexpected fault\n"
813 "\tAddr: 0x%08lx\\n"
814 "\tAddress space: %s\n"
815 "\tSource ID: %d\n"
816 "\tType: %d\n",
817 fault_reg & PAGE_MASK,
818 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
819 RING_FAULT_SRCID(fault_reg),
820 RING_FAULT_FAULT_TYPE(fault_reg));
821 I915_WRITE(RING_FAULT_REG(ring),
822 fault_reg & ~RING_FAULT_VALID);
823 }
824 }
825 POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
826}
827
828void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
829{
830 struct drm_i915_private *dev_priv = dev->dev_private;
831
832 /* Don't bother messing with faults pre GEN6 as we have little
833 * documentation supporting that it's a good idea.
834 */
835 if (INTEL_INFO(dev)->gen < 6)
836 return;
837
838 i915_check_and_clear_faults(dev);
839
840 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
841 dev_priv->gtt.base.start / PAGE_SIZE,
842 dev_priv->gtt.base.total / PAGE_SIZE,
843 false);
844}
845
Daniel Vetter76aaf222010-11-05 22:23:30 +0100846void i915_gem_restore_gtt_mappings(struct drm_device *dev)
847{
848 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson05394f32010-11-08 19:18:58 +0000849 struct drm_i915_gem_object *obj;
Daniel Vetter76aaf222010-11-05 22:23:30 +0100850
Ben Widawsky828c7902013-10-16 09:21:30 -0700851 i915_check_and_clear_faults(dev);
852
Chris Wilsonbee4a182011-01-21 10:54:32 +0000853 /* First fill our portion of the GTT with scratch pages */
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700854 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
855 dev_priv->gtt.base.start / PAGE_SIZE,
Ben Widawsky828c7902013-10-16 09:21:30 -0700856 dev_priv->gtt.base.total / PAGE_SIZE,
857 true);
Chris Wilsonbee4a182011-01-21 10:54:32 +0000858
Ben Widawsky35c20a62013-05-31 11:28:48 -0700859 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
Chris Wilson2c225692013-08-09 12:26:45 +0100860 i915_gem_clflush_object(obj, obj->pin_display);
Daniel Vetter74163902012-02-15 23:50:21 +0100861 i915_gem_gtt_bind_object(obj, obj->cache_level);
Daniel Vetter76aaf222010-11-05 22:23:30 +0100862 }
863
Ben Widawskye76e9ae2012-11-04 09:21:27 -0800864 i915_gem_chipset_flush(dev);
Daniel Vetter76aaf222010-11-05 22:23:30 +0100865}
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +0100866
Daniel Vetter74163902012-02-15 23:50:21 +0100867int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +0100868{
Chris Wilson9da3da62012-06-01 15:20:22 +0100869 if (obj->has_dma_mapping)
Daniel Vetter74163902012-02-15 23:50:21 +0100870 return 0;
Chris Wilson9da3da62012-06-01 15:20:22 +0100871
872 if (!dma_map_sg(&obj->base.dev->pdev->dev,
873 obj->pages->sgl, obj->pages->nents,
874 PCI_DMA_BIDIRECTIONAL))
875 return -ENOSPC;
876
877 return 0;
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +0100878}
879
Ben Widawsky94ec8f62013-11-02 21:07:18 -0700880static inline void gen8_set_pte(void __iomem *addr, gen8_gtt_pte_t pte)
881{
882#ifdef writeq
883 writeq(pte, addr);
884#else
885 iowrite32((u32)pte, addr);
886 iowrite32(pte >> 32, addr + 4);
887#endif
888}
889
890static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
891 struct sg_table *st,
892 unsigned int first_entry,
893 enum i915_cache_level level)
894{
895 struct drm_i915_private *dev_priv = vm->dev->dev_private;
896 gen8_gtt_pte_t __iomem *gtt_entries =
897 (gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
898 int i = 0;
899 struct sg_page_iter sg_iter;
900 dma_addr_t addr;
901
902 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
903 addr = sg_dma_address(sg_iter.sg) +
904 (sg_iter.sg_pgoffset << PAGE_SHIFT);
905 gen8_set_pte(&gtt_entries[i],
906 gen8_pte_encode(addr, level, true));
907 i++;
908 }
909
910 /*
911 * XXX: This serves as a posting read to make sure that the PTE has
912 * actually been updated. There is some concern that even though
913 * registers and PTEs are within the same BAR that they are potentially
914 * of NUMA access patterns. Therefore, even with the way we assume
915 * hardware should work, we must keep this posting read for paranoia.
916 */
917 if (i != 0)
918 WARN_ON(readq(&gtt_entries[i-1])
919 != gen8_pte_encode(addr, level, true));
920
921#if 0 /* TODO: Still needed on GEN8? */
922 /* This next bit makes the above posting read even more important. We
923 * want to flush the TLBs only after we're certain all the PTE updates
924 * have finished.
925 */
926 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
927 POSTING_READ(GFX_FLSH_CNTL_GEN6);
928#endif
929}
930
Ben Widawskye76e9ae2012-11-04 09:21:27 -0800931/*
932 * Binds an object into the global gtt with the specified cache level. The object
933 * will be accessible to the GPU via commands whose operands reference offsets
934 * within the global GTT as well as accessible by the GPU through the GMADR
935 * mapped BAR (dev_priv->mm.gtt->gtt).
936 */
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700937static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
Daniel Vetter7faf1ab2013-01-24 14:44:55 -0800938 struct sg_table *st,
939 unsigned int first_entry,
940 enum i915_cache_level level)
Ben Widawskye76e9ae2012-11-04 09:21:27 -0800941{
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700942 struct drm_i915_private *dev_priv = vm->dev->dev_private;
Ben Widawskye7c2b582013-04-08 18:43:48 -0700943 gen6_gtt_pte_t __iomem *gtt_entries =
944 (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
Imre Deak6e995e22013-02-18 19:28:04 +0200945 int i = 0;
946 struct sg_page_iter sg_iter;
Ben Widawskye76e9ae2012-11-04 09:21:27 -0800947 dma_addr_t addr;
948
Imre Deak6e995e22013-02-18 19:28:04 +0200949 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
Imre Deak2db76d72013-03-26 15:14:18 +0200950 addr = sg_page_iter_dma_address(&sg_iter);
Ben Widawskyb35b3802013-10-16 09:18:21 -0700951 iowrite32(vm->pte_encode(addr, level, true), &gtt_entries[i]);
Imre Deak6e995e22013-02-18 19:28:04 +0200952 i++;
Ben Widawskye76e9ae2012-11-04 09:21:27 -0800953 }
954
Ben Widawskye76e9ae2012-11-04 09:21:27 -0800955 /* XXX: This serves as a posting read to make sure that the PTE has
956 * actually been updated. There is some concern that even though
957 * registers and PTEs are within the same BAR that they are potentially
958 * of NUMA access patterns. Therefore, even with the way we assume
959 * hardware should work, we must keep this posting read for paranoia.
960 */
961 if (i != 0)
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700962 WARN_ON(readl(&gtt_entries[i-1]) !=
Ben Widawskyb35b3802013-10-16 09:18:21 -0700963 vm->pte_encode(addr, level, true));
Ben Widawsky0f9b91c2012-11-04 09:21:30 -0800964
965 /* This next bit makes the above posting read even more important. We
966 * want to flush the TLBs only after we're certain all the PTE updates
967 * have finished.
968 */
969 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
970 POSTING_READ(GFX_FLSH_CNTL_GEN6);
Ben Widawskye76e9ae2012-11-04 09:21:27 -0800971}
972
Ben Widawsky94ec8f62013-11-02 21:07:18 -0700973static void gen8_ggtt_clear_range(struct i915_address_space *vm,
974 unsigned int first_entry,
975 unsigned int num_entries,
976 bool use_scratch)
977{
978 struct drm_i915_private *dev_priv = vm->dev->dev_private;
979 gen8_gtt_pte_t scratch_pte, __iomem *gtt_base =
980 (gen8_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
981 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
982 int i;
983
984 if (WARN(num_entries > max_entries,
985 "First entry = %d; Num entries = %d (max=%d)\n",
986 first_entry, num_entries, max_entries))
987 num_entries = max_entries;
988
989 scratch_pte = gen8_pte_encode(vm->scratch.addr,
990 I915_CACHE_LLC,
991 use_scratch);
992 for (i = 0; i < num_entries; i++)
993 gen8_set_pte(&gtt_base[i], scratch_pte);
994 readl(gtt_base);
995}
996
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700997static void gen6_ggtt_clear_range(struct i915_address_space *vm,
Daniel Vetter7faf1ab2013-01-24 14:44:55 -0800998 unsigned int first_entry,
Ben Widawsky828c7902013-10-16 09:21:30 -0700999 unsigned int num_entries,
1000 bool use_scratch)
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001001{
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001002 struct drm_i915_private *dev_priv = vm->dev->dev_private;
Ben Widawskye7c2b582013-04-08 18:43:48 -07001003 gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
1004 (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
Ben Widawskya54c0c22013-01-24 14:45:00 -08001005 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001006 int i;
1007
1008 if (WARN(num_entries > max_entries,
1009 "First entry = %d; Num entries = %d (max=%d)\n",
1010 first_entry, num_entries, max_entries))
1011 num_entries = max_entries;
1012
Ben Widawsky828c7902013-10-16 09:21:30 -07001013 scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch);
1014
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001015 for (i = 0; i < num_entries; i++)
1016 iowrite32(scratch_pte, &gtt_base[i]);
1017 readl(gtt_base);
1018}
1019
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001020static void i915_ggtt_insert_entries(struct i915_address_space *vm,
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001021 struct sg_table *st,
1022 unsigned int pg_start,
1023 enum i915_cache_level cache_level)
1024{
1025 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
1026 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
1027
1028 intel_gtt_insert_sg_entries(st, pg_start, flags);
1029
1030}
1031
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001032static void i915_ggtt_clear_range(struct i915_address_space *vm,
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001033 unsigned int first_entry,
Ben Widawsky828c7902013-10-16 09:21:30 -07001034 unsigned int num_entries,
1035 bool unused)
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001036{
1037 intel_gtt_clear_range(first_entry, num_entries);
1038}
1039
1040
Daniel Vetter74163902012-02-15 23:50:21 +01001041void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
1042 enum i915_cache_level cache_level)
Chris Wilsond5bd1442011-04-14 06:48:26 +01001043{
1044 struct drm_device *dev = obj->base.dev;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001045 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001046 const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001047
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001048 dev_priv->gtt.base.insert_entries(&dev_priv->gtt.base, obj->pages,
1049 entry,
1050 cache_level);
Chris Wilsond5bd1442011-04-14 06:48:26 +01001051
Daniel Vetter74898d72012-02-15 23:50:22 +01001052 obj->has_global_gtt_mapping = 1;
Chris Wilsond5bd1442011-04-14 06:48:26 +01001053}
1054
Chris Wilson05394f32010-11-08 19:18:58 +00001055void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01001056{
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001057 struct drm_device *dev = obj->base.dev;
1058 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001059 const unsigned long entry = i915_gem_obj_ggtt_offset(obj) >> PAGE_SHIFT;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001060
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001061 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
1062 entry,
Ben Widawsky828c7902013-10-16 09:21:30 -07001063 obj->base.size >> PAGE_SHIFT,
1064 true);
Daniel Vetter74898d72012-02-15 23:50:22 +01001065
1066 obj->has_global_gtt_mapping = 0;
Daniel Vetter74163902012-02-15 23:50:21 +01001067}
1068
1069void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
1070{
Ben Widawsky5c042282011-10-17 15:51:55 -07001071 struct drm_device *dev = obj->base.dev;
1072 struct drm_i915_private *dev_priv = dev->dev_private;
1073 bool interruptible;
1074
1075 interruptible = do_idling(dev_priv);
1076
Chris Wilson9da3da62012-06-01 15:20:22 +01001077 if (!obj->has_dma_mapping)
1078 dma_unmap_sg(&dev->pdev->dev,
1079 obj->pages->sgl, obj->pages->nents,
1080 PCI_DMA_BIDIRECTIONAL);
Ben Widawsky5c042282011-10-17 15:51:55 -07001081
1082 undo_idling(dev_priv, interruptible);
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01001083}
Daniel Vetter644ec022012-03-26 09:45:40 +02001084
Chris Wilson42d6ab42012-07-26 11:49:32 +01001085static void i915_gtt_color_adjust(struct drm_mm_node *node,
1086 unsigned long color,
1087 unsigned long *start,
1088 unsigned long *end)
1089{
1090 if (node->color != color)
1091 *start += 4096;
1092
1093 if (!list_empty(&node->node_list)) {
1094 node = list_entry(node->node_list.next,
1095 struct drm_mm_node,
1096 node_list);
1097 if (node->allocated && node->color != color)
1098 *end -= 4096;
1099 }
1100}
Ben Widawskyfbe5d362013-11-04 19:56:49 -08001101
Ben Widawskyd7e50082012-12-18 10:31:25 -08001102void i915_gem_setup_global_gtt(struct drm_device *dev,
1103 unsigned long start,
1104 unsigned long mappable_end,
1105 unsigned long end)
Daniel Vetter644ec022012-03-26 09:45:40 +02001106{
Ben Widawskye78891c2013-01-25 16:41:04 -08001107 /* Let GEM Manage all of the aperture.
1108 *
1109 * However, leave one page at the end still bound to the scratch page.
1110 * There are a number of places where the hardware apparently prefetches
1111 * past the end of the object, and we've seen multiple hangs with the
1112 * GPU head pointer stuck in a batchbuffer bound at the last page of the
1113 * aperture. One page should be enough to keep any prefetching inside
1114 * of the aperture.
1115 */
Ben Widawsky40d749802013-07-31 16:59:59 -07001116 struct drm_i915_private *dev_priv = dev->dev_private;
1117 struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
Chris Wilsoned2f3452012-11-15 11:32:19 +00001118 struct drm_mm_node *entry;
1119 struct drm_i915_gem_object *obj;
1120 unsigned long hole_start, hole_end;
Daniel Vetter644ec022012-03-26 09:45:40 +02001121
Ben Widawsky35451cb2013-01-17 12:45:13 -08001122 BUG_ON(mappable_end > end);
1123
Chris Wilsoned2f3452012-11-15 11:32:19 +00001124 /* Subtract the guard page ... */
Ben Widawsky40d749802013-07-31 16:59:59 -07001125 drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
Chris Wilson42d6ab42012-07-26 11:49:32 +01001126 if (!HAS_LLC(dev))
Ben Widawsky93bd8642013-07-16 16:50:06 -07001127 dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
Daniel Vetter644ec022012-03-26 09:45:40 +02001128
Chris Wilsoned2f3452012-11-15 11:32:19 +00001129 /* Mark any preallocated objects as occupied */
Ben Widawsky35c20a62013-05-31 11:28:48 -07001130 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
Ben Widawsky40d749802013-07-31 16:59:59 -07001131 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
Ben Widawskyb3a070c2013-07-05 14:41:02 -07001132 int ret;
Ben Widawskyedd41a82013-07-05 14:41:05 -07001133 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
Ben Widawskyc6cfb322013-07-05 14:41:06 -07001134 i915_gem_obj_ggtt_offset(obj), obj->base.size);
Chris Wilsoned2f3452012-11-15 11:32:19 +00001135
Ben Widawskyc6cfb322013-07-05 14:41:06 -07001136 WARN_ON(i915_gem_obj_ggtt_bound(obj));
Ben Widawsky40d749802013-07-31 16:59:59 -07001137 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
Ben Widawskyc6cfb322013-07-05 14:41:06 -07001138 if (ret)
Ben Widawskyb3a070c2013-07-05 14:41:02 -07001139 DRM_DEBUG_KMS("Reservation failed\n");
Chris Wilsoned2f3452012-11-15 11:32:19 +00001140 obj->has_global_gtt_mapping = 1;
1141 }
1142
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001143 dev_priv->gtt.base.start = start;
1144 dev_priv->gtt.base.total = end - start;
Daniel Vetter644ec022012-03-26 09:45:40 +02001145
Chris Wilsoned2f3452012-11-15 11:32:19 +00001146 /* Clear any non-preallocated blocks */
Ben Widawsky40d749802013-07-31 16:59:59 -07001147 drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001148 const unsigned long count = (hole_end - hole_start) / PAGE_SIZE;
Chris Wilsoned2f3452012-11-15 11:32:19 +00001149 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
1150 hole_start, hole_end);
Ben Widawsky828c7902013-10-16 09:21:30 -07001151 ggtt_vm->clear_range(ggtt_vm, hole_start / PAGE_SIZE, count, true);
Chris Wilsoned2f3452012-11-15 11:32:19 +00001152 }
1153
1154 /* And finally clear the reserved guard page */
Ben Widawsky828c7902013-10-16 09:21:30 -07001155 ggtt_vm->clear_range(ggtt_vm, end / PAGE_SIZE - 1, 1, true);
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001156}
1157
Ben Widawskyd7e50082012-12-18 10:31:25 -08001158static bool
1159intel_enable_ppgtt(struct drm_device *dev)
1160{
1161 if (i915_enable_ppgtt >= 0)
1162 return i915_enable_ppgtt;
1163
1164#ifdef CONFIG_INTEL_IOMMU
1165 /* Disable ppgtt on SNB if VT-d is on. */
1166 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
1167 return false;
1168#endif
1169
1170 return true;
1171}
1172
1173void i915_gem_init_global_gtt(struct drm_device *dev)
1174{
1175 struct drm_i915_private *dev_priv = dev->dev_private;
1176 unsigned long gtt_size, mappable_size;
Ben Widawskyd7e50082012-12-18 10:31:25 -08001177
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001178 gtt_size = dev_priv->gtt.base.total;
Ben Widawsky93d18792013-01-17 12:45:17 -08001179 mappable_size = dev_priv->gtt.mappable_end;
Ben Widawskyd7e50082012-12-18 10:31:25 -08001180
1181 if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
Ben Widawskye78891c2013-01-25 16:41:04 -08001182 int ret;
Ben Widawsky3eb1c002013-04-08 18:43:52 -07001183
1184 if (INTEL_INFO(dev)->gen <= 7) {
1185 /* PPGTT pdes are stolen from global gtt ptes, so shrink the
1186 * aperture accordingly when using aliasing ppgtt. */
Ben Widawsky6670a5a2013-06-27 16:30:04 -07001187 gtt_size -= GEN6_PPGTT_PD_ENTRIES * PAGE_SIZE;
Ben Widawsky3eb1c002013-04-08 18:43:52 -07001188 }
Ben Widawskyd7e50082012-12-18 10:31:25 -08001189
1190 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
1191
1192 ret = i915_gem_init_aliasing_ppgtt(dev);
Ben Widawskye78891c2013-01-25 16:41:04 -08001193 if (!ret)
Ben Widawskyd7e50082012-12-18 10:31:25 -08001194 return;
Ben Widawskye78891c2013-01-25 16:41:04 -08001195
1196 DRM_ERROR("Aliased PPGTT setup failed %d\n", ret);
Ben Widawsky93bd8642013-07-16 16:50:06 -07001197 drm_mm_takedown(&dev_priv->gtt.base.mm);
Ville Syrjäläb42218c2013-11-02 21:07:29 -07001198 if (INTEL_INFO(dev)->gen < 8)
1199 gtt_size += GEN6_PPGTT_PD_ENTRIES*PAGE_SIZE;
Ben Widawskyd7e50082012-12-18 10:31:25 -08001200 }
Ben Widawskye78891c2013-01-25 16:41:04 -08001201 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001202}
1203
1204static int setup_scratch_page(struct drm_device *dev)
1205{
1206 struct drm_i915_private *dev_priv = dev->dev_private;
1207 struct page *page;
1208 dma_addr_t dma_addr;
1209
1210 page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
1211 if (page == NULL)
1212 return -ENOMEM;
1213 get_page(page);
1214 set_pages_uc(page, 1);
1215
1216#ifdef CONFIG_INTEL_IOMMU
1217 dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
1218 PCI_DMA_BIDIRECTIONAL);
1219 if (pci_dma_mapping_error(dev->pdev, dma_addr))
1220 return -EINVAL;
1221#else
1222 dma_addr = page_to_phys(page);
1223#endif
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001224 dev_priv->gtt.base.scratch.page = page;
1225 dev_priv->gtt.base.scratch.addr = dma_addr;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001226
1227 return 0;
1228}
1229
1230static void teardown_scratch_page(struct drm_device *dev)
1231{
1232 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001233 struct page *page = dev_priv->gtt.base.scratch.page;
1234
1235 set_pages_wb(page, 1);
1236 pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001237 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001238 put_page(page);
1239 __free_page(page);
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001240}
1241
1242static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
1243{
1244 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
1245 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
1246 return snb_gmch_ctl << 20;
1247}
1248
Ben Widawsky9459d252013-11-03 16:53:55 -08001249static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
1250{
1251 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
1252 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
1253 if (bdw_gmch_ctl)
1254 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
Ben Widawsky3a2ffb62013-11-07 21:40:51 -08001255 if (bdw_gmch_ctl > 4) {
1256 WARN_ON(!i915_preliminary_hw_support);
1257 return 4<<20;
1258 }
1259
Ben Widawsky9459d252013-11-03 16:53:55 -08001260 return bdw_gmch_ctl << 20;
1261}
1262
Ben Widawskybaa09f52013-01-24 13:49:57 -08001263static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001264{
1265 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
1266 snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
1267 return snb_gmch_ctl << 25; /* 32 MB units */
1268}
1269
Ben Widawsky9459d252013-11-03 16:53:55 -08001270static inline size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
1271{
1272 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
1273 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
1274 return bdw_gmch_ctl << 25; /* 32 MB units */
1275}
1276
Ben Widawsky63340132013-11-04 19:32:22 -08001277static int ggtt_probe_common(struct drm_device *dev,
1278 size_t gtt_size)
1279{
1280 struct drm_i915_private *dev_priv = dev->dev_private;
1281 phys_addr_t gtt_bus_addr;
1282 int ret;
1283
1284 /* For Modern GENs the PTEs and register space are split in the BAR */
1285 gtt_bus_addr = pci_resource_start(dev->pdev, 0) +
1286 (pci_resource_len(dev->pdev, 0) / 2);
1287
1288 dev_priv->gtt.gsm = ioremap_wc(gtt_bus_addr, gtt_size);
1289 if (!dev_priv->gtt.gsm) {
1290 DRM_ERROR("Failed to map the gtt page table\n");
1291 return -ENOMEM;
1292 }
1293
1294 ret = setup_scratch_page(dev);
1295 if (ret) {
1296 DRM_ERROR("Scratch setup failed\n");
1297 /* iounmap will also get called at remove, but meh */
1298 iounmap(dev_priv->gtt.gsm);
1299 }
1300
1301 return ret;
1302}
1303
Ben Widawskyfbe5d362013-11-04 19:56:49 -08001304/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
1305 * bits. When using advanced contexts each context stores its own PAT, but
1306 * writing this data shouldn't be harmful even in those cases. */
1307static void gen8_setup_private_ppat(struct drm_i915_private *dev_priv)
1308{
1309#define GEN8_PPAT_UC (0<<0)
1310#define GEN8_PPAT_WC (1<<0)
1311#define GEN8_PPAT_WT (2<<0)
1312#define GEN8_PPAT_WB (3<<0)
1313#define GEN8_PPAT_ELLC_OVERRIDE (0<<2)
1314/* FIXME(BDW): Bspec is completely confused about cache control bits. */
1315#define GEN8_PPAT_LLC (1<<2)
1316#define GEN8_PPAT_LLCELLC (2<<2)
1317#define GEN8_PPAT_LLCeLLC (3<<2)
1318#define GEN8_PPAT_AGE(x) (x<<4)
1319#define GEN8_PPAT(i, x) ((uint64_t) (x) << ((i) * 8))
1320 uint64_t pat;
1321
1322 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */
1323 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
1324 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
1325 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */
1326 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
1327 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
1328 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
1329 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
1330
1331 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
1332 * write would work. */
1333 I915_WRITE(GEN8_PRIVATE_PAT, pat);
1334 I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
1335}
1336
Ben Widawsky63340132013-11-04 19:32:22 -08001337static int gen8_gmch_probe(struct drm_device *dev,
1338 size_t *gtt_total,
1339 size_t *stolen,
1340 phys_addr_t *mappable_base,
1341 unsigned long *mappable_end)
1342{
1343 struct drm_i915_private *dev_priv = dev->dev_private;
1344 unsigned int gtt_size;
1345 u16 snb_gmch_ctl;
1346 int ret;
1347
1348 /* TODO: We're not aware of mappable constraints on gen8 yet */
1349 *mappable_base = pci_resource_start(dev->pdev, 2);
1350 *mappable_end = pci_resource_len(dev->pdev, 2);
1351
1352 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
1353 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
1354
1355 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1356
1357 *stolen = gen8_get_stolen_size(snb_gmch_ctl);
1358
1359 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
Ben Widawskyd31eb102013-11-02 21:07:17 -07001360 *gtt_total = (gtt_size / sizeof(gen8_gtt_pte_t)) << PAGE_SHIFT;
Ben Widawsky63340132013-11-04 19:32:22 -08001361
Ben Widawskyfbe5d362013-11-04 19:56:49 -08001362 gen8_setup_private_ppat(dev_priv);
1363
Ben Widawsky63340132013-11-04 19:32:22 -08001364 ret = ggtt_probe_common(dev, gtt_size);
1365
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001366 dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
1367 dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
Ben Widawsky63340132013-11-04 19:32:22 -08001368
1369 return ret;
1370}
1371
Ben Widawskybaa09f52013-01-24 13:49:57 -08001372static int gen6_gmch_probe(struct drm_device *dev,
1373 size_t *gtt_total,
Ben Widawsky41907dd2013-02-08 11:32:47 -08001374 size_t *stolen,
1375 phys_addr_t *mappable_base,
1376 unsigned long *mappable_end)
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001377{
1378 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawskybaa09f52013-01-24 13:49:57 -08001379 unsigned int gtt_size;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001380 u16 snb_gmch_ctl;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001381 int ret;
1382
Ben Widawsky41907dd2013-02-08 11:32:47 -08001383 *mappable_base = pci_resource_start(dev->pdev, 2);
1384 *mappable_end = pci_resource_len(dev->pdev, 2);
1385
Ben Widawskybaa09f52013-01-24 13:49:57 -08001386 /* 64/512MB is the current min/max we actually know of, but this is just
1387 * a coarse sanity check.
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001388 */
Ben Widawsky41907dd2013-02-08 11:32:47 -08001389 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
Ben Widawskybaa09f52013-01-24 13:49:57 -08001390 DRM_ERROR("Unknown GMADR size (%lx)\n",
1391 dev_priv->gtt.mappable_end);
1392 return -ENXIO;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001393 }
1394
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001395 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
1396 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
Ben Widawskybaa09f52013-01-24 13:49:57 -08001397 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
Ben Widawskybaa09f52013-01-24 13:49:57 -08001398
Ben Widawskyc4ae25e2013-05-01 11:00:34 -07001399 *stolen = gen6_get_stolen_size(snb_gmch_ctl);
Ben Widawskybaa09f52013-01-24 13:49:57 -08001400
Ben Widawsky63340132013-11-04 19:32:22 -08001401 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
Ben Widawskybaa09f52013-01-24 13:49:57 -08001402 *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
1403
Ben Widawsky63340132013-11-04 19:32:22 -08001404 ret = ggtt_probe_common(dev, gtt_size);
Ben Widawskybaa09f52013-01-24 13:49:57 -08001405
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001406 dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
1407 dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
Ben Widawskybaa09f52013-01-24 13:49:57 -08001408
1409 return ret;
1410}
1411
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001412static void gen6_gmch_remove(struct i915_address_space *vm)
Ben Widawskybaa09f52013-01-24 13:49:57 -08001413{
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001414
1415 struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
Ben Widawsky5ed16782013-11-25 09:54:43 -08001416
1417 drm_mm_takedown(&vm->mm);
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001418 iounmap(gtt->gsm);
1419 teardown_scratch_page(vm->dev);
Ben Widawskybaa09f52013-01-24 13:49:57 -08001420}
1421
1422static int i915_gmch_probe(struct drm_device *dev,
1423 size_t *gtt_total,
Ben Widawsky41907dd2013-02-08 11:32:47 -08001424 size_t *stolen,
1425 phys_addr_t *mappable_base,
1426 unsigned long *mappable_end)
Ben Widawskybaa09f52013-01-24 13:49:57 -08001427{
1428 struct drm_i915_private *dev_priv = dev->dev_private;
1429 int ret;
1430
Ben Widawskybaa09f52013-01-24 13:49:57 -08001431 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
1432 if (!ret) {
1433 DRM_ERROR("failed to set up gmch\n");
1434 return -EIO;
1435 }
1436
Ben Widawsky41907dd2013-02-08 11:32:47 -08001437 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
Ben Widawskybaa09f52013-01-24 13:49:57 -08001438
1439 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001440 dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
1441 dev_priv->gtt.base.insert_entries = i915_ggtt_insert_entries;
Ben Widawskybaa09f52013-01-24 13:49:57 -08001442
Chris Wilsonc0a7f812013-12-30 12:16:15 +00001443 if (unlikely(dev_priv->gtt.do_idle_maps))
1444 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
1445
Ben Widawskybaa09f52013-01-24 13:49:57 -08001446 return 0;
1447}
1448
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001449static void i915_gmch_remove(struct i915_address_space *vm)
Ben Widawskybaa09f52013-01-24 13:49:57 -08001450{
1451 intel_gmch_remove();
1452}
1453
1454int i915_gem_gtt_init(struct drm_device *dev)
1455{
1456 struct drm_i915_private *dev_priv = dev->dev_private;
1457 struct i915_gtt *gtt = &dev_priv->gtt;
Ben Widawskybaa09f52013-01-24 13:49:57 -08001458 int ret;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001459
Ben Widawskybaa09f52013-01-24 13:49:57 -08001460 if (INTEL_INFO(dev)->gen <= 5) {
Ben Widawskyb2f21b42013-06-27 16:30:20 -07001461 gtt->gtt_probe = i915_gmch_probe;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001462 gtt->base.cleanup = i915_gmch_remove;
Ben Widawsky63340132013-11-04 19:32:22 -08001463 } else if (INTEL_INFO(dev)->gen < 8) {
Ben Widawskyb2f21b42013-06-27 16:30:20 -07001464 gtt->gtt_probe = gen6_gmch_probe;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001465 gtt->base.cleanup = gen6_gmch_remove;
Ben Widawsky4d15c142013-07-04 11:02:06 -07001466 if (IS_HASWELL(dev) && dev_priv->ellc_size)
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001467 gtt->base.pte_encode = iris_pte_encode;
Ben Widawsky4d15c142013-07-04 11:02:06 -07001468 else if (IS_HASWELL(dev))
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001469 gtt->base.pte_encode = hsw_pte_encode;
Ben Widawskyb2f21b42013-06-27 16:30:20 -07001470 else if (IS_VALLEYVIEW(dev))
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001471 gtt->base.pte_encode = byt_pte_encode;
Chris Wilson350ec882013-08-06 13:17:02 +01001472 else if (INTEL_INFO(dev)->gen >= 7)
1473 gtt->base.pte_encode = ivb_pte_encode;
Ben Widawskyb2f21b42013-06-27 16:30:20 -07001474 else
Chris Wilson350ec882013-08-06 13:17:02 +01001475 gtt->base.pte_encode = snb_pte_encode;
Ben Widawsky63340132013-11-04 19:32:22 -08001476 } else {
1477 dev_priv->gtt.gtt_probe = gen8_gmch_probe;
1478 dev_priv->gtt.base.cleanup = gen6_gmch_remove;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001479 }
1480
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001481 ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
Ben Widawskyb2f21b42013-06-27 16:30:20 -07001482 &gtt->mappable_base, &gtt->mappable_end);
Ben Widawskya54c0c22013-01-24 14:45:00 -08001483 if (ret)
Ben Widawskybaa09f52013-01-24 13:49:57 -08001484 return ret;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001485
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001486 gtt->base.dev = dev;
1487
Ben Widawskybaa09f52013-01-24 13:49:57 -08001488 /* GMADR is the PCI mmio aperture into the global GTT. */
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001489 DRM_INFO("Memory usable by graphics device = %zdM\n",
1490 gtt->base.total >> 20);
Ben Widawskyb2f21b42013-06-27 16:30:20 -07001491 DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
1492 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001493
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001494 return 0;
Daniel Vetter644ec022012-03-26 09:45:40 +02001495}