blob: 94916362b61c816fb92db377bcd526db1be91661 [file] [log] [blame]
Daniel Vetter76aaf222010-11-05 22:23:30 +01001/*
2 * Copyright © 2010 Daniel Vetter
Ben Widawskyc4ac5242014-02-19 22:05:47 -08003 * Copyright © 2011-2014 Intel Corporation
Daniel Vetter76aaf222010-11-05 22:23:30 +01004 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 *
24 */
25
Daniel Vetter0e46ce22014-01-08 16:10:27 +010026#include <linux/seq_file.h>
David Howells760285e2012-10-02 18:01:07 +010027#include <drm/drmP.h>
28#include <drm/i915_drm.h>
Daniel Vetter76aaf222010-11-05 22:23:30 +010029#include "i915_drv.h"
30#include "i915_trace.h"
31#include "intel_drv.h"
32
Ville Syrjäläee0ce472014-04-09 13:28:01 +030033static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv);
34static void chv_setup_private_ppat(struct drm_i915_private *dev_priv);
Ben Widawskya2319c02014-03-18 16:09:37 -070035
Daniel Vetter93a25a92014-03-06 09:40:43 +010036bool intel_enable_ppgtt(struct drm_device *dev, bool full)
37{
38 if (i915.enable_ppgtt == 0 || !HAS_ALIASING_PPGTT(dev))
39 return false;
40
41 if (i915.enable_ppgtt == 1 && full)
42 return false;
43
44#ifdef CONFIG_INTEL_IOMMU
45 /* Disable ppgtt on SNB if VT-d is on. */
46 if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
47 DRM_INFO("Disabling PPGTT because VT-d is on\n");
48 return false;
49 }
50#endif
51
52 /* Full ppgtt disabled by default for now due to issues. */
53 if (full)
Ben Widawsky8d214b72014-03-24 18:06:00 -070054 return HAS_PPGTT(dev) && (i915.enable_ppgtt == 2);
Daniel Vetter93a25a92014-03-06 09:40:43 +010055 else
56 return HAS_ALIASING_PPGTT(dev);
57}
58
Ben Widawskyfbe5d362013-11-04 19:56:49 -080059
Ben Widawsky6f65e292013-12-06 14:10:56 -080060static void ppgtt_bind_vma(struct i915_vma *vma,
61 enum i915_cache_level cache_level,
62 u32 flags);
63static void ppgtt_unbind_vma(struct i915_vma *vma);
Ben Widawskyeeb94882013-12-06 14:11:10 -080064static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt);
Ben Widawsky6f65e292013-12-06 14:10:56 -080065
Ben Widawsky94ec8f62013-11-02 21:07:18 -070066static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
67 enum i915_cache_level level,
68 bool valid)
69{
70 gen8_gtt_pte_t pte = valid ? _PAGE_PRESENT | _PAGE_RW : 0;
71 pte |= addr;
Ben Widawsky63c42e52014-04-18 18:04:27 -030072
73 switch (level) {
74 case I915_CACHE_NONE:
Ben Widawskyfbe5d362013-11-04 19:56:49 -080075 pte |= PPAT_UNCACHED_INDEX;
Ben Widawsky63c42e52014-04-18 18:04:27 -030076 break;
77 case I915_CACHE_WT:
78 pte |= PPAT_DISPLAY_ELLC_INDEX;
79 break;
80 default:
81 pte |= PPAT_CACHED_INDEX;
82 break;
83 }
84
Ben Widawsky94ec8f62013-11-02 21:07:18 -070085 return pte;
86}
87
Ben Widawskyb1fe6672013-11-04 21:20:14 -080088static inline gen8_ppgtt_pde_t gen8_pde_encode(struct drm_device *dev,
89 dma_addr_t addr,
90 enum i915_cache_level level)
91{
92 gen8_ppgtt_pde_t pde = _PAGE_PRESENT | _PAGE_RW;
93 pde |= addr;
94 if (level != I915_CACHE_NONE)
95 pde |= PPAT_CACHED_PDE_INDEX;
96 else
97 pde |= PPAT_UNCACHED_INDEX;
98 return pde;
99}
100
Chris Wilson350ec882013-08-06 13:17:02 +0100101static gen6_gtt_pte_t snb_pte_encode(dma_addr_t addr,
Ben Widawskyb35b3802013-10-16 09:18:21 -0700102 enum i915_cache_level level,
103 bool valid)
Ben Widawsky54d12522012-09-24 16:44:32 -0700104{
Ben Widawskyb35b3802013-10-16 09:18:21 -0700105 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Ben Widawsky54d12522012-09-24 16:44:32 -0700106 pte |= GEN6_PTE_ADDR_ENCODE(addr);
Ben Widawskye7210c32012-10-19 09:33:22 -0700107
108 switch (level) {
Chris Wilson350ec882013-08-06 13:17:02 +0100109 case I915_CACHE_L3_LLC:
110 case I915_CACHE_LLC:
111 pte |= GEN6_PTE_CACHE_LLC;
112 break;
113 case I915_CACHE_NONE:
114 pte |= GEN6_PTE_UNCACHED;
115 break;
116 default:
117 WARN_ON(1);
118 }
119
120 return pte;
121}
122
123static gen6_gtt_pte_t ivb_pte_encode(dma_addr_t addr,
Ben Widawskyb35b3802013-10-16 09:18:21 -0700124 enum i915_cache_level level,
125 bool valid)
Chris Wilson350ec882013-08-06 13:17:02 +0100126{
Ben Widawskyb35b3802013-10-16 09:18:21 -0700127 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Chris Wilson350ec882013-08-06 13:17:02 +0100128 pte |= GEN6_PTE_ADDR_ENCODE(addr);
129
130 switch (level) {
131 case I915_CACHE_L3_LLC:
132 pte |= GEN7_PTE_CACHE_L3_LLC;
Ben Widawskye7210c32012-10-19 09:33:22 -0700133 break;
134 case I915_CACHE_LLC:
135 pte |= GEN6_PTE_CACHE_LLC;
136 break;
137 case I915_CACHE_NONE:
Kenneth Graunke91197082013-04-22 00:53:51 -0700138 pte |= GEN6_PTE_UNCACHED;
Ben Widawskye7210c32012-10-19 09:33:22 -0700139 break;
140 default:
Chris Wilson350ec882013-08-06 13:17:02 +0100141 WARN_ON(1);
Ben Widawskye7210c32012-10-19 09:33:22 -0700142 }
143
Ben Widawsky54d12522012-09-24 16:44:32 -0700144 return pte;
145}
146
Ben Widawsky80a74f72013-06-27 16:30:19 -0700147static gen6_gtt_pte_t byt_pte_encode(dma_addr_t addr,
Ben Widawskyb35b3802013-10-16 09:18:21 -0700148 enum i915_cache_level level,
149 bool valid)
Kenneth Graunke93c34e72013-04-22 00:53:50 -0700150{
Ben Widawskyb35b3802013-10-16 09:18:21 -0700151 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Kenneth Graunke93c34e72013-04-22 00:53:50 -0700152 pte |= GEN6_PTE_ADDR_ENCODE(addr);
153
154 /* Mark the page as writeable. Other platforms don't have a
155 * setting for read-only/writable, so this matches that behavior.
156 */
157 pte |= BYT_PTE_WRITEABLE;
158
159 if (level != I915_CACHE_NONE)
160 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES;
161
162 return pte;
163}
164
Ben Widawsky80a74f72013-06-27 16:30:19 -0700165static gen6_gtt_pte_t hsw_pte_encode(dma_addr_t addr,
Ben Widawskyb35b3802013-10-16 09:18:21 -0700166 enum i915_cache_level level,
167 bool valid)
Kenneth Graunke91197082013-04-22 00:53:51 -0700168{
Ben Widawskyb35b3802013-10-16 09:18:21 -0700169 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Ben Widawsky0d8ff152013-07-04 11:02:03 -0700170 pte |= HSW_PTE_ADDR_ENCODE(addr);
Kenneth Graunke91197082013-04-22 00:53:51 -0700171
172 if (level != I915_CACHE_NONE)
Ben Widawsky87a6b682013-08-04 23:47:29 -0700173 pte |= HSW_WB_LLC_AGE3;
Kenneth Graunke91197082013-04-22 00:53:51 -0700174
175 return pte;
176}
177
Ben Widawsky4d15c142013-07-04 11:02:06 -0700178static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
Ben Widawskyb35b3802013-10-16 09:18:21 -0700179 enum i915_cache_level level,
180 bool valid)
Ben Widawsky4d15c142013-07-04 11:02:06 -0700181{
Ben Widawskyb35b3802013-10-16 09:18:21 -0700182 gen6_gtt_pte_t pte = valid ? GEN6_PTE_VALID : 0;
Ben Widawsky4d15c142013-07-04 11:02:06 -0700183 pte |= HSW_PTE_ADDR_ENCODE(addr);
184
Chris Wilson651d7942013-08-08 14:41:10 +0100185 switch (level) {
186 case I915_CACHE_NONE:
187 break;
188 case I915_CACHE_WT:
Chris Wilsonc51e9702013-11-22 10:37:53 +0000189 pte |= HSW_WT_ELLC_LLC_AGE3;
Chris Wilson651d7942013-08-08 14:41:10 +0100190 break;
191 default:
Chris Wilsonc51e9702013-11-22 10:37:53 +0000192 pte |= HSW_WB_ELLC_LLC_AGE3;
Chris Wilson651d7942013-08-08 14:41:10 +0100193 break;
194 }
Ben Widawsky4d15c142013-07-04 11:02:06 -0700195
196 return pte;
197}
198
Ben Widawsky94e409c2013-11-04 22:29:36 -0800199/* Broadwell Page Directory Pointer Descriptors */
Oscar Mateoa4872ba2014-05-22 14:13:33 +0100200static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,
Ben Widawskye178f702013-12-06 14:10:47 -0800201 uint64_t val, bool synchronous)
Ben Widawsky94e409c2013-11-04 22:29:36 -0800202{
Ben Widawskye178f702013-12-06 14:10:47 -0800203 struct drm_i915_private *dev_priv = ring->dev->dev_private;
Ben Widawsky94e409c2013-11-04 22:29:36 -0800204 int ret;
205
206 BUG_ON(entry >= 4);
207
Ben Widawskye178f702013-12-06 14:10:47 -0800208 if (synchronous) {
209 I915_WRITE(GEN8_RING_PDP_UDW(ring, entry), val >> 32);
210 I915_WRITE(GEN8_RING_PDP_LDW(ring, entry), (u32)val);
211 return 0;
212 }
213
Ben Widawsky94e409c2013-11-04 22:29:36 -0800214 ret = intel_ring_begin(ring, 6);
215 if (ret)
216 return ret;
217
218 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
219 intel_ring_emit(ring, GEN8_RING_PDP_UDW(ring, entry));
220 intel_ring_emit(ring, (u32)(val >> 32));
221 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
222 intel_ring_emit(ring, GEN8_RING_PDP_LDW(ring, entry));
223 intel_ring_emit(ring, (u32)(val));
224 intel_ring_advance(ring);
225
226 return 0;
227}
228
Ben Widawskyeeb94882013-12-06 14:11:10 -0800229static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
Oscar Mateoa4872ba2014-05-22 14:13:33 +0100230 struct intel_engine_cs *ring,
Ben Widawskyeeb94882013-12-06 14:11:10 -0800231 bool synchronous)
Ben Widawsky94e409c2013-11-04 22:29:36 -0800232{
Ben Widawskyeeb94882013-12-06 14:11:10 -0800233 int i, ret;
Ben Widawsky94e409c2013-11-04 22:29:36 -0800234
235 /* bit of a hack to find the actual last used pd */
236 int used_pd = ppgtt->num_pd_entries / GEN8_PDES_PER_PAGE;
237
Ben Widawsky94e409c2013-11-04 22:29:36 -0800238 for (i = used_pd - 1; i >= 0; i--) {
239 dma_addr_t addr = ppgtt->pd_dma_addr[i];
Ben Widawskyeeb94882013-12-06 14:11:10 -0800240 ret = gen8_write_pdp(ring, i, addr, synchronous);
241 if (ret)
242 return ret;
Ben Widawsky94e409c2013-11-04 22:29:36 -0800243 }
Ben Widawskyd595bd42013-11-25 09:54:32 -0800244
Ben Widawskyeeb94882013-12-06 14:11:10 -0800245 return 0;
Ben Widawsky94e409c2013-11-04 22:29:36 -0800246}
247
Ben Widawsky459108b2013-11-02 21:07:23 -0700248static void gen8_ppgtt_clear_range(struct i915_address_space *vm,
Ben Widawsky782f1492014-02-20 11:50:33 -0800249 uint64_t start,
250 uint64_t length,
Ben Widawsky459108b2013-11-02 21:07:23 -0700251 bool use_scratch)
252{
253 struct i915_hw_ppgtt *ppgtt =
254 container_of(vm, struct i915_hw_ppgtt, base);
255 gen8_gtt_pte_t *pt_vaddr, scratch_pte;
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800256 unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
257 unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
258 unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
Ben Widawsky782f1492014-02-20 11:50:33 -0800259 unsigned num_entries = length >> PAGE_SHIFT;
Ben Widawsky459108b2013-11-02 21:07:23 -0700260 unsigned last_pte, i;
261
262 scratch_pte = gen8_pte_encode(ppgtt->base.scratch.addr,
263 I915_CACHE_LLC, use_scratch);
264
265 while (num_entries) {
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800266 struct page *page_table = ppgtt->gen8_pt_pages[pdpe][pde];
Ben Widawsky459108b2013-11-02 21:07:23 -0700267
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800268 last_pte = pte + num_entries;
Ben Widawsky459108b2013-11-02 21:07:23 -0700269 if (last_pte > GEN8_PTES_PER_PAGE)
270 last_pte = GEN8_PTES_PER_PAGE;
271
272 pt_vaddr = kmap_atomic(page_table);
273
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800274 for (i = pte; i < last_pte; i++) {
Ben Widawsky459108b2013-11-02 21:07:23 -0700275 pt_vaddr[i] = scratch_pte;
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800276 num_entries--;
277 }
Ben Widawsky459108b2013-11-02 21:07:23 -0700278
Rafael Barbalhofd1ab8f2014-04-09 13:28:02 +0300279 if (!HAS_LLC(ppgtt->base.dev))
280 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
Ben Widawsky459108b2013-11-02 21:07:23 -0700281 kunmap_atomic(pt_vaddr);
282
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800283 pte = 0;
284 if (++pde == GEN8_PDES_PER_PAGE) {
285 pdpe++;
286 pde = 0;
287 }
Ben Widawsky459108b2013-11-02 21:07:23 -0700288 }
289}
290
Ben Widawsky9df15b42013-11-02 21:07:24 -0700291static void gen8_ppgtt_insert_entries(struct i915_address_space *vm,
292 struct sg_table *pages,
Ben Widawsky782f1492014-02-20 11:50:33 -0800293 uint64_t start,
Ben Widawsky9df15b42013-11-02 21:07:24 -0700294 enum i915_cache_level cache_level)
295{
296 struct i915_hw_ppgtt *ppgtt =
297 container_of(vm, struct i915_hw_ppgtt, base);
298 gen8_gtt_pte_t *pt_vaddr;
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800299 unsigned pdpe = start >> GEN8_PDPE_SHIFT & GEN8_PDPE_MASK;
300 unsigned pde = start >> GEN8_PDE_SHIFT & GEN8_PDE_MASK;
301 unsigned pte = start >> GEN8_PTE_SHIFT & GEN8_PTE_MASK;
Ben Widawsky9df15b42013-11-02 21:07:24 -0700302 struct sg_page_iter sg_iter;
303
Chris Wilson6f1cc992013-12-31 15:50:31 +0000304 pt_vaddr = NULL;
Ben Widawsky9df15b42013-11-02 21:07:24 -0700305
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800306 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
307 if (WARN_ON(pdpe >= GEN8_LEGACY_PDPS))
308 break;
309
310 if (pt_vaddr == NULL)
311 pt_vaddr = kmap_atomic(ppgtt->gen8_pt_pages[pdpe][pde]);
312
313 pt_vaddr[pte] =
Chris Wilson6f1cc992013-12-31 15:50:31 +0000314 gen8_pte_encode(sg_page_iter_dma_address(&sg_iter),
315 cache_level, true);
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800316 if (++pte == GEN8_PTES_PER_PAGE) {
Rafael Barbalhofd1ab8f2014-04-09 13:28:02 +0300317 if (!HAS_LLC(ppgtt->base.dev))
318 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
Ben Widawsky9df15b42013-11-02 21:07:24 -0700319 kunmap_atomic(pt_vaddr);
Chris Wilson6f1cc992013-12-31 15:50:31 +0000320 pt_vaddr = NULL;
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800321 if (++pde == GEN8_PDES_PER_PAGE) {
322 pdpe++;
323 pde = 0;
324 }
325 pte = 0;
Ben Widawsky9df15b42013-11-02 21:07:24 -0700326 }
327 }
Rafael Barbalhofd1ab8f2014-04-09 13:28:02 +0300328 if (pt_vaddr) {
329 if (!HAS_LLC(ppgtt->base.dev))
330 drm_clflush_virt_range(pt_vaddr, PAGE_SIZE);
Chris Wilson6f1cc992013-12-31 15:50:31 +0000331 kunmap_atomic(pt_vaddr);
Rafael Barbalhofd1ab8f2014-04-09 13:28:02 +0300332 }
Ben Widawsky9df15b42013-11-02 21:07:24 -0700333}
334
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800335static void gen8_free_page_tables(struct page **pt_pages)
Ben Widawskyb45a6712014-02-12 14:28:44 -0800336{
337 int i;
338
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800339 if (pt_pages == NULL)
340 return;
Ben Widawskyb45a6712014-02-12 14:28:44 -0800341
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800342 for (i = 0; i < GEN8_PDES_PER_PAGE; i++)
343 if (pt_pages[i])
344 __free_pages(pt_pages[i], 0);
345}
346
347static void gen8_ppgtt_free(const struct i915_hw_ppgtt *ppgtt)
348{
349 int i;
350
351 for (i = 0; i < ppgtt->num_pd_pages; i++) {
352 gen8_free_page_tables(ppgtt->gen8_pt_pages[i]);
353 kfree(ppgtt->gen8_pt_pages[i]);
354 kfree(ppgtt->gen8_pt_dma_addr[i]);
355 }
356
Ben Widawskyb45a6712014-02-12 14:28:44 -0800357 __free_pages(ppgtt->pd_pages, get_order(ppgtt->num_pd_pages << PAGE_SHIFT));
358}
359
360static void gen8_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
361{
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800362 struct pci_dev *hwdev = ppgtt->base.dev->pdev;
Ben Widawskyb45a6712014-02-12 14:28:44 -0800363 int i, j;
364
365 for (i = 0; i < ppgtt->num_pd_pages; i++) {
366 /* TODO: In the future we'll support sparse mappings, so this
367 * will have to change. */
368 if (!ppgtt->pd_dma_addr[i])
369 continue;
370
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800371 pci_unmap_page(hwdev, ppgtt->pd_dma_addr[i], PAGE_SIZE,
372 PCI_DMA_BIDIRECTIONAL);
Ben Widawskyb45a6712014-02-12 14:28:44 -0800373
374 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
375 dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
376 if (addr)
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800377 pci_unmap_page(hwdev, addr, PAGE_SIZE,
378 PCI_DMA_BIDIRECTIONAL);
Ben Widawskyb45a6712014-02-12 14:28:44 -0800379 }
380 }
381}
382
Ben Widawsky37aca442013-11-04 20:47:32 -0800383static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
384{
385 struct i915_hw_ppgtt *ppgtt =
386 container_of(vm, struct i915_hw_ppgtt, base);
Ben Widawsky37aca442013-11-04 20:47:32 -0800387
Ben Widawsky7e0d96b2013-12-06 14:11:26 -0800388 list_del(&vm->global_link);
Ben Widawsky686e1f62013-11-25 09:54:34 -0800389 drm_mm_takedown(&vm->mm);
390
Ben Widawskyb45a6712014-02-12 14:28:44 -0800391 gen8_ppgtt_unmap_pages(ppgtt);
392 gen8_ppgtt_free(ppgtt);
Ben Widawsky37aca442013-11-04 20:47:32 -0800393}
394
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800395static struct page **__gen8_alloc_page_tables(void)
396{
397 struct page **pt_pages;
398 int i;
399
400 pt_pages = kcalloc(GEN8_PDES_PER_PAGE, sizeof(struct page *), GFP_KERNEL);
401 if (!pt_pages)
402 return ERR_PTR(-ENOMEM);
403
404 for (i = 0; i < GEN8_PDES_PER_PAGE; i++) {
405 pt_pages[i] = alloc_page(GFP_KERNEL);
406 if (!pt_pages[i])
407 goto bail;
408 }
409
410 return pt_pages;
411
412bail:
413 gen8_free_page_tables(pt_pages);
414 kfree(pt_pages);
415 return ERR_PTR(-ENOMEM);
416}
417
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800418static int gen8_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt,
419 const int max_pdp)
420{
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800421 struct page **pt_pages[GEN8_LEGACY_PDPS];
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800422 int i, ret;
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800423
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800424 for (i = 0; i < max_pdp; i++) {
425 pt_pages[i] = __gen8_alloc_page_tables();
426 if (IS_ERR(pt_pages[i])) {
427 ret = PTR_ERR(pt_pages[i]);
428 goto unwind_out;
429 }
430 }
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800431
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800432 /* NB: Avoid touching gen8_pt_pages until last to keep the allocation,
433 * "atomic" - for cleanup purposes.
434 */
435 for (i = 0; i < max_pdp; i++)
436 ppgtt->gen8_pt_pages[i] = pt_pages[i];
437
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800438 return 0;
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800439
440unwind_out:
441 while (i--) {
442 gen8_free_page_tables(pt_pages[i]);
443 kfree(pt_pages[i]);
444 }
445
446 return ret;
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800447}
448
449static int gen8_ppgtt_allocate_dma(struct i915_hw_ppgtt *ppgtt)
450{
451 int i;
452
453 for (i = 0; i < ppgtt->num_pd_pages; i++) {
454 ppgtt->gen8_pt_dma_addr[i] = kcalloc(GEN8_PDES_PER_PAGE,
455 sizeof(dma_addr_t),
456 GFP_KERNEL);
457 if (!ppgtt->gen8_pt_dma_addr[i])
458 return -ENOMEM;
459 }
460
461 return 0;
462}
463
464static int gen8_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt,
465 const int max_pdp)
466{
467 ppgtt->pd_pages = alloc_pages(GFP_KERNEL, get_order(max_pdp << PAGE_SHIFT));
468 if (!ppgtt->pd_pages)
469 return -ENOMEM;
470
471 ppgtt->num_pd_pages = 1 << get_order(max_pdp << PAGE_SHIFT);
472 BUG_ON(ppgtt->num_pd_pages > GEN8_LEGACY_PDPS);
473
474 return 0;
475}
476
477static int gen8_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt,
478 const int max_pdp)
479{
480 int ret;
481
482 ret = gen8_ppgtt_allocate_page_directories(ppgtt, max_pdp);
483 if (ret)
484 return ret;
485
486 ret = gen8_ppgtt_allocate_page_tables(ppgtt, max_pdp);
487 if (ret) {
488 __free_pages(ppgtt->pd_pages, get_order(max_pdp << PAGE_SHIFT));
489 return ret;
490 }
491
492 ppgtt->num_pd_entries = max_pdp * GEN8_PDES_PER_PAGE;
493
494 ret = gen8_ppgtt_allocate_dma(ppgtt);
495 if (ret)
496 gen8_ppgtt_free(ppgtt);
497
498 return ret;
499}
500
501static int gen8_ppgtt_setup_page_directories(struct i915_hw_ppgtt *ppgtt,
502 const int pd)
503{
504 dma_addr_t pd_addr;
505 int ret;
506
507 pd_addr = pci_map_page(ppgtt->base.dev->pdev,
508 &ppgtt->pd_pages[pd], 0,
509 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
510
511 ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pd_addr);
512 if (ret)
513 return ret;
514
515 ppgtt->pd_dma_addr[pd] = pd_addr;
516
517 return 0;
518}
519
520static int gen8_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt,
521 const int pd,
522 const int pt)
523{
524 dma_addr_t pt_addr;
525 struct page *p;
526 int ret;
527
Ben Widawsky7ad47cf2014-02-20 11:51:21 -0800528 p = ppgtt->gen8_pt_pages[pd][pt];
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800529 pt_addr = pci_map_page(ppgtt->base.dev->pdev,
530 p, 0, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
531 ret = pci_dma_mapping_error(ppgtt->base.dev->pdev, pt_addr);
532 if (ret)
533 return ret;
534
535 ppgtt->gen8_pt_dma_addr[pd][pt] = pt_addr;
536
537 return 0;
538}
539
Ben Widawsky37aca442013-11-04 20:47:32 -0800540/**
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800541 * GEN8 legacy ppgtt programming is accomplished through a max 4 PDP registers
542 * with a net effect resembling a 2-level page table in normal x86 terms. Each
543 * PDP represents 1GB of memory 4 * 512 * 512 * 4096 = 4GB legacy 32b address
544 * space.
Ben Widawsky37aca442013-11-04 20:47:32 -0800545 *
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800546 * FIXME: split allocation into smaller pieces. For now we only ever do this
547 * once, but with full PPGTT, the multiple contiguous allocations will be bad.
Ben Widawsky37aca442013-11-04 20:47:32 -0800548 * TODO: Do something with the size parameter
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800549 */
Ben Widawsky37aca442013-11-04 20:47:32 -0800550static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
551{
Ben Widawsky37aca442013-11-04 20:47:32 -0800552 const int max_pdp = DIV_ROUND_UP(size, 1 << 30);
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800553 const int min_pt_pages = GEN8_PDES_PER_PAGE * max_pdp;
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800554 int i, j, ret;
Ben Widawsky37aca442013-11-04 20:47:32 -0800555
556 if (size % (1<<30))
557 DRM_INFO("Pages will be wasted unless GTT size (%llu) is divisible by 1GB\n", size);
558
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800559 /* 1. Do all our allocations for page directories and page tables. */
560 ret = gen8_ppgtt_alloc(ppgtt, max_pdp);
561 if (ret)
562 return ret;
Ben Widawsky37aca442013-11-04 20:47:32 -0800563
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800564 /*
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800565 * 2. Create DMA mappings for the page directories and page tables.
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800566 */
567 for (i = 0; i < max_pdp; i++) {
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800568 ret = gen8_ppgtt_setup_page_directories(ppgtt, i);
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800569 if (ret)
570 goto bail;
571
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800572 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
Ben Widawskybf2b4ed2014-02-19 22:05:43 -0800573 ret = gen8_ppgtt_setup_page_tables(ppgtt, i, j);
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800574 if (ret)
575 goto bail;
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800576 }
577 }
578
579 /*
580 * 3. Map all the page directory entires to point to the page tables
581 * we've allocated.
582 *
583 * For now, the PPGTT helper functions all require that the PDEs are
Ben Widawskyb1fe6672013-11-04 21:20:14 -0800584 * plugged in correctly. So we do that now/here. For aliasing PPGTT, we
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800585 * will never need to touch the PDEs again.
586 */
Ben Widawskyb1fe6672013-11-04 21:20:14 -0800587 for (i = 0; i < max_pdp; i++) {
588 gen8_ppgtt_pde_t *pd_vaddr;
589 pd_vaddr = kmap_atomic(&ppgtt->pd_pages[i]);
590 for (j = 0; j < GEN8_PDES_PER_PAGE; j++) {
591 dma_addr_t addr = ppgtt->gen8_pt_dma_addr[i][j];
592 pd_vaddr[j] = gen8_pde_encode(ppgtt->base.dev, addr,
593 I915_CACHE_LLC);
594 }
Rafael Barbalhofd1ab8f2014-04-09 13:28:02 +0300595 if (!HAS_LLC(ppgtt->base.dev))
596 drm_clflush_virt_range(pd_vaddr, PAGE_SIZE);
Ben Widawskyb1fe6672013-11-04 21:20:14 -0800597 kunmap_atomic(pd_vaddr);
598 }
599
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800600 ppgtt->enable = gen8_ppgtt_enable;
601 ppgtt->switch_mm = gen8_mm_switch;
602 ppgtt->base.clear_range = gen8_ppgtt_clear_range;
603 ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
604 ppgtt->base.cleanup = gen8_ppgtt_cleanup;
605 ppgtt->base.start = 0;
Ben Widawsky5abbcca2014-02-21 13:06:34 -0800606 ppgtt->base.total = ppgtt->num_pd_entries * GEN8_PTES_PER_PAGE * PAGE_SIZE;
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800607
Ben Widawsky5abbcca2014-02-21 13:06:34 -0800608 ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
Ben Widawsky459108b2013-11-02 21:07:23 -0700609
Ben Widawsky37aca442013-11-04 20:47:32 -0800610 DRM_DEBUG_DRIVER("Allocated %d pages for page directories (%d wasted)\n",
611 ppgtt->num_pd_pages, ppgtt->num_pd_pages - max_pdp);
612 DRM_DEBUG_DRIVER("Allocated %d pages for page tables (%lld wasted)\n",
Ben Widawsky5abbcca2014-02-21 13:06:34 -0800613 ppgtt->num_pd_entries,
614 (ppgtt->num_pd_entries - min_pt_pages) + size % (1<<30));
Ben Widawsky28cf5412013-11-02 21:07:26 -0700615 return 0;
Ben Widawsky37aca442013-11-04 20:47:32 -0800616
Ben Widawskyf3a964b2014-02-19 22:05:42 -0800617bail:
618 gen8_ppgtt_unmap_pages(ppgtt);
619 gen8_ppgtt_free(ppgtt);
Ben Widawsky37aca442013-11-04 20:47:32 -0800620 return ret;
621}
622
Ben Widawsky87d60b62013-12-06 14:11:29 -0800623static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
624{
625 struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
626 struct i915_address_space *vm = &ppgtt->base;
627 gen6_gtt_pte_t __iomem *pd_addr;
628 gen6_gtt_pte_t scratch_pte;
629 uint32_t pd_entry;
630 int pte, pde;
631
632 scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true);
633
634 pd_addr = (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm +
635 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
636
637 seq_printf(m, " VM %p (pd_offset %x-%x):\n", vm,
638 ppgtt->pd_offset, ppgtt->pd_offset + ppgtt->num_pd_entries);
639 for (pde = 0; pde < ppgtt->num_pd_entries; pde++) {
640 u32 expected;
641 gen6_gtt_pte_t *pt_vaddr;
642 dma_addr_t pt_addr = ppgtt->pt_dma_addr[pde];
643 pd_entry = readl(pd_addr + pde);
644 expected = (GEN6_PDE_ADDR_ENCODE(pt_addr) | GEN6_PDE_VALID);
645
646 if (pd_entry != expected)
647 seq_printf(m, "\tPDE #%d mismatch: Actual PDE: %x Expected PDE: %x\n",
648 pde,
649 pd_entry,
650 expected);
651 seq_printf(m, "\tPDE: %x\n", pd_entry);
652
653 pt_vaddr = kmap_atomic(ppgtt->pt_pages[pde]);
654 for (pte = 0; pte < I915_PPGTT_PT_ENTRIES; pte+=4) {
655 unsigned long va =
656 (pde * PAGE_SIZE * I915_PPGTT_PT_ENTRIES) +
657 (pte * PAGE_SIZE);
658 int i;
659 bool found = false;
660 for (i = 0; i < 4; i++)
661 if (pt_vaddr[pte + i] != scratch_pte)
662 found = true;
663 if (!found)
664 continue;
665
666 seq_printf(m, "\t\t0x%lx [%03d,%04d]: =", va, pde, pte);
667 for (i = 0; i < 4; i++) {
668 if (pt_vaddr[pte + i] != scratch_pte)
669 seq_printf(m, " %08x", pt_vaddr[pte + i]);
670 else
671 seq_puts(m, " SCRATCH ");
672 }
673 seq_puts(m, "\n");
674 }
675 kunmap_atomic(pt_vaddr);
676 }
677}
678
Ben Widawsky3e302542013-04-23 23:15:32 -0700679static void gen6_write_pdes(struct i915_hw_ppgtt *ppgtt)
Ben Widawsky61973492013-04-08 18:43:54 -0700680{
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700681 struct drm_i915_private *dev_priv = ppgtt->base.dev->dev_private;
Ben Widawsky61973492013-04-08 18:43:54 -0700682 gen6_gtt_pte_t __iomem *pd_addr;
683 uint32_t pd_entry;
684 int i;
685
Ben Widawsky0a732872013-04-23 23:15:30 -0700686 WARN_ON(ppgtt->pd_offset & 0x3f);
Ben Widawsky61973492013-04-08 18:43:54 -0700687 pd_addr = (gen6_gtt_pte_t __iomem*)dev_priv->gtt.gsm +
688 ppgtt->pd_offset / sizeof(gen6_gtt_pte_t);
689 for (i = 0; i < ppgtt->num_pd_entries; i++) {
690 dma_addr_t pt_addr;
691
692 pt_addr = ppgtt->pt_dma_addr[i];
693 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr);
694 pd_entry |= GEN6_PDE_VALID;
695
696 writel(pd_entry, pd_addr + i);
697 }
698 readl(pd_addr);
Ben Widawsky3e302542013-04-23 23:15:32 -0700699}
700
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800701static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
Ben Widawsky3e302542013-04-23 23:15:32 -0700702{
Ben Widawsky3e302542013-04-23 23:15:32 -0700703 BUG_ON(ppgtt->pd_offset & 0x3f);
704
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800705 return (ppgtt->pd_offset / 64) << 16;
706}
Ben Widawsky61973492013-04-08 18:43:54 -0700707
Ben Widawsky90252e52013-12-06 14:11:12 -0800708static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
Oscar Mateoa4872ba2014-05-22 14:13:33 +0100709 struct intel_engine_cs *ring,
Ben Widawsky90252e52013-12-06 14:11:12 -0800710 bool synchronous)
711{
712 struct drm_device *dev = ppgtt->base.dev;
713 struct drm_i915_private *dev_priv = dev->dev_private;
714 int ret;
Ben Widawsky61973492013-04-08 18:43:54 -0700715
Ben Widawsky90252e52013-12-06 14:11:12 -0800716 /* If we're in reset, we can assume the GPU is sufficiently idle to
717 * manually frob these bits. Ideally we could use the ring functions,
718 * except our error handling makes it quite difficult (can't use
719 * intel_ring_begin, ring->flush, or intel_ring_advance)
720 *
721 * FIXME: We should try not to special case reset
722 */
723 if (synchronous ||
724 i915_reset_in_progress(&dev_priv->gpu_error)) {
725 WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt);
726 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
727 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
728 POSTING_READ(RING_PP_DIR_BASE(ring));
729 return 0;
Ben Widawsky61973492013-04-08 18:43:54 -0700730 }
731
Ben Widawsky90252e52013-12-06 14:11:12 -0800732 /* NB: TLBs must be flushed and invalidated before a switch */
733 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
734 if (ret)
735 return ret;
736
737 ret = intel_ring_begin(ring, 6);
738 if (ret)
739 return ret;
740
741 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
742 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
743 intel_ring_emit(ring, PP_DIR_DCLV_2G);
744 intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
745 intel_ring_emit(ring, get_pd_offset(ppgtt));
746 intel_ring_emit(ring, MI_NOOP);
747 intel_ring_advance(ring);
748
749 return 0;
750}
751
Ben Widawsky48a10382013-12-06 14:11:11 -0800752static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
Oscar Mateoa4872ba2014-05-22 14:13:33 +0100753 struct intel_engine_cs *ring,
Ben Widawsky48a10382013-12-06 14:11:11 -0800754 bool synchronous)
755{
756 struct drm_device *dev = ppgtt->base.dev;
757 struct drm_i915_private *dev_priv = dev->dev_private;
758 int ret;
759
760 /* If we're in reset, we can assume the GPU is sufficiently idle to
761 * manually frob these bits. Ideally we could use the ring functions,
762 * except our error handling makes it quite difficult (can't use
763 * intel_ring_begin, ring->flush, or intel_ring_advance)
764 *
765 * FIXME: We should try not to special case reset
766 */
767 if (synchronous ||
768 i915_reset_in_progress(&dev_priv->gpu_error)) {
769 WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt);
770 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
771 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
772 POSTING_READ(RING_PP_DIR_BASE(ring));
773 return 0;
774 }
775
776 /* NB: TLBs must be flushed and invalidated before a switch */
777 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
778 if (ret)
779 return ret;
780
781 ret = intel_ring_begin(ring, 6);
782 if (ret)
783 return ret;
784
785 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(2));
786 intel_ring_emit(ring, RING_PP_DIR_DCLV(ring));
787 intel_ring_emit(ring, PP_DIR_DCLV_2G);
788 intel_ring_emit(ring, RING_PP_DIR_BASE(ring));
789 intel_ring_emit(ring, get_pd_offset(ppgtt));
790 intel_ring_emit(ring, MI_NOOP);
791 intel_ring_advance(ring);
792
Ben Widawsky90252e52013-12-06 14:11:12 -0800793 /* XXX: RCS is the only one to auto invalidate the TLBs? */
794 if (ring->id != RCS) {
795 ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
796 if (ret)
797 return ret;
798 }
799
Ben Widawsky48a10382013-12-06 14:11:11 -0800800 return 0;
801}
802
Ben Widawskyeeb94882013-12-06 14:11:10 -0800803static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
Oscar Mateoa4872ba2014-05-22 14:13:33 +0100804 struct intel_engine_cs *ring,
Ben Widawskyeeb94882013-12-06 14:11:10 -0800805 bool synchronous)
806{
807 struct drm_device *dev = ppgtt->base.dev;
808 struct drm_i915_private *dev_priv = dev->dev_private;
809
Ben Widawsky48a10382013-12-06 14:11:11 -0800810 if (!synchronous)
811 return 0;
812
Ben Widawskyeeb94882013-12-06 14:11:10 -0800813 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
814 I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
815
816 POSTING_READ(RING_PP_DIR_DCLV(ring));
817
818 return 0;
819}
820
821static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
822{
823 struct drm_device *dev = ppgtt->base.dev;
824 struct drm_i915_private *dev_priv = dev->dev_private;
Oscar Mateoa4872ba2014-05-22 14:13:33 +0100825 struct intel_engine_cs *ring;
Ben Widawskyeeb94882013-12-06 14:11:10 -0800826 int j, ret;
827
828 for_each_ring(ring, dev_priv, j) {
829 I915_WRITE(RING_MODE_GEN7(ring),
830 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
Ben Widawskyd2ff7192013-12-06 14:11:27 -0800831
832 /* We promise to do a switch later with FULL PPGTT. If this is
833 * aliasing, this is the one and only switch we'll do */
834 if (USES_FULL_PPGTT(dev))
835 continue;
836
Ben Widawskyeeb94882013-12-06 14:11:10 -0800837 ret = ppgtt->switch_mm(ppgtt, ring, true);
838 if (ret)
839 goto err_out;
840 }
841
842 return 0;
843
844err_out:
845 for_each_ring(ring, dev_priv, j)
846 I915_WRITE(RING_MODE_GEN7(ring),
847 _MASKED_BIT_DISABLE(GFX_PPGTT_ENABLE));
848 return ret;
849}
850
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800851static int gen7_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
852{
853 struct drm_device *dev = ppgtt->base.dev;
Jani Nikula50227e12014-03-31 14:27:21 +0300854 struct drm_i915_private *dev_priv = dev->dev_private;
Oscar Mateoa4872ba2014-05-22 14:13:33 +0100855 struct intel_engine_cs *ring;
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800856 uint32_t ecochk, ecobits;
857 int i;
858
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800859 ecobits = I915_READ(GAC_ECO_BITS);
860 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
861
862 ecochk = I915_READ(GAM_ECOCHK);
863 if (IS_HASWELL(dev)) {
864 ecochk |= ECOCHK_PPGTT_WB_HSW;
865 } else {
866 ecochk |= ECOCHK_PPGTT_LLC_IVB;
867 ecochk &= ~ECOCHK_PPGTT_GFDT_IVB;
868 }
869 I915_WRITE(GAM_ECOCHK, ecochk);
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800870
Ben Widawsky61973492013-04-08 18:43:54 -0700871 for_each_ring(ring, dev_priv, i) {
Ben Widawskyeeb94882013-12-06 14:11:10 -0800872 int ret;
873 /* GFX_MODE is per-ring on gen7+ */
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800874 I915_WRITE(RING_MODE_GEN7(ring),
875 _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
Ben Widawsky61973492013-04-08 18:43:54 -0700876
Ben Widawskyd2ff7192013-12-06 14:11:27 -0800877 /* We promise to do a switch later with FULL PPGTT. If this is
878 * aliasing, this is the one and only switch we'll do */
879 if (USES_FULL_PPGTT(dev))
880 continue;
881
Ben Widawskyeeb94882013-12-06 14:11:10 -0800882 ret = ppgtt->switch_mm(ppgtt, ring, true);
883 if (ret)
884 return ret;
Ben Widawsky61973492013-04-08 18:43:54 -0700885 }
Ben Widawskyd2ff7192013-12-06 14:11:27 -0800886
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800887 return 0;
888}
889
Ben Widawskya3d67d22013-12-06 14:11:06 -0800890static int gen6_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
Ben Widawsky61973492013-04-08 18:43:54 -0700891{
Ben Widawskya3d67d22013-12-06 14:11:06 -0800892 struct drm_device *dev = ppgtt->base.dev;
Jani Nikula50227e12014-03-31 14:27:21 +0300893 struct drm_i915_private *dev_priv = dev->dev_private;
Oscar Mateoa4872ba2014-05-22 14:13:33 +0100894 struct intel_engine_cs *ring;
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800895 uint32_t ecochk, gab_ctl, ecobits;
Ben Widawsky61973492013-04-08 18:43:54 -0700896 int i;
897
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800898 ecobits = I915_READ(GAC_ECO_BITS);
899 I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
900 ECOBITS_PPGTT_CACHE64B);
Ben Widawsky61973492013-04-08 18:43:54 -0700901
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800902 gab_ctl = I915_READ(GAB_CTL);
903 I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
Ben Widawsky61973492013-04-08 18:43:54 -0700904
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800905 ecochk = I915_READ(GAM_ECOCHK);
906 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
Ben Widawsky61973492013-04-08 18:43:54 -0700907
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800908 I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
Ben Widawsky61973492013-04-08 18:43:54 -0700909
910 for_each_ring(ring, dev_priv, i) {
Ben Widawskyeeb94882013-12-06 14:11:10 -0800911 int ret = ppgtt->switch_mm(ppgtt, ring, true);
912 if (ret)
913 return ret;
Ben Widawsky61973492013-04-08 18:43:54 -0700914 }
Ben Widawskyb4a74e32013-12-06 14:11:09 -0800915
Ben Widawskyb7c36d22013-04-08 18:43:56 -0700916 return 0;
Ben Widawsky61973492013-04-08 18:43:54 -0700917}
918
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100919/* PPGTT support for Sandybdrige/Gen6 and later */
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700920static void gen6_ppgtt_clear_range(struct i915_address_space *vm,
Ben Widawsky782f1492014-02-20 11:50:33 -0800921 uint64_t start,
922 uint64_t length,
Ben Widawsky828c7902013-10-16 09:21:30 -0700923 bool use_scratch)
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100924{
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700925 struct i915_hw_ppgtt *ppgtt =
926 container_of(vm, struct i915_hw_ppgtt, base);
Ben Widawskye7c2b582013-04-08 18:43:48 -0700927 gen6_gtt_pte_t *pt_vaddr, scratch_pte;
Ben Widawsky782f1492014-02-20 11:50:33 -0800928 unsigned first_entry = start >> PAGE_SHIFT;
929 unsigned num_entries = length >> PAGE_SHIFT;
Daniel Vettera15326a2013-03-19 23:48:39 +0100930 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
Daniel Vetter7bddb012012-02-09 17:15:47 +0100931 unsigned first_pte = first_entry % I915_PPGTT_PT_ENTRIES;
932 unsigned last_pte, i;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100933
Ben Widawskyb35b3802013-10-16 09:18:21 -0700934 scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, true);
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100935
Daniel Vetter7bddb012012-02-09 17:15:47 +0100936 while (num_entries) {
937 last_pte = first_pte + num_entries;
938 if (last_pte > I915_PPGTT_PT_ENTRIES)
939 last_pte = I915_PPGTT_PT_ENTRIES;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100940
Daniel Vettera15326a2013-03-19 23:48:39 +0100941 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
Daniel Vetter7bddb012012-02-09 17:15:47 +0100942
943 for (i = first_pte; i < last_pte; i++)
944 pt_vaddr[i] = scratch_pte;
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100945
946 kunmap_atomic(pt_vaddr);
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100947
Daniel Vetter7bddb012012-02-09 17:15:47 +0100948 num_entries -= last_pte - first_pte;
949 first_pte = 0;
Daniel Vettera15326a2013-03-19 23:48:39 +0100950 act_pt++;
Daniel Vetter7bddb012012-02-09 17:15:47 +0100951 }
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100952}
953
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700954static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
Daniel Vetterdef886c2013-01-24 14:44:56 -0800955 struct sg_table *pages,
Ben Widawsky782f1492014-02-20 11:50:33 -0800956 uint64_t start,
Daniel Vetterdef886c2013-01-24 14:44:56 -0800957 enum i915_cache_level cache_level)
958{
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700959 struct i915_hw_ppgtt *ppgtt =
960 container_of(vm, struct i915_hw_ppgtt, base);
Ben Widawskye7c2b582013-04-08 18:43:48 -0700961 gen6_gtt_pte_t *pt_vaddr;
Ben Widawsky782f1492014-02-20 11:50:33 -0800962 unsigned first_entry = start >> PAGE_SHIFT;
Daniel Vettera15326a2013-03-19 23:48:39 +0100963 unsigned act_pt = first_entry / I915_PPGTT_PT_ENTRIES;
Imre Deak6e995e22013-02-18 19:28:04 +0200964 unsigned act_pte = first_entry % I915_PPGTT_PT_ENTRIES;
965 struct sg_page_iter sg_iter;
Daniel Vetterdef886c2013-01-24 14:44:56 -0800966
Chris Wilsoncc797142013-12-31 15:50:30 +0000967 pt_vaddr = NULL;
Imre Deak6e995e22013-02-18 19:28:04 +0200968 for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
Chris Wilsoncc797142013-12-31 15:50:30 +0000969 if (pt_vaddr == NULL)
970 pt_vaddr = kmap_atomic(ppgtt->pt_pages[act_pt]);
Daniel Vetterdef886c2013-01-24 14:44:56 -0800971
Chris Wilsoncc797142013-12-31 15:50:30 +0000972 pt_vaddr[act_pte] =
973 vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
974 cache_level, true);
Imre Deak6e995e22013-02-18 19:28:04 +0200975 if (++act_pte == I915_PPGTT_PT_ENTRIES) {
976 kunmap_atomic(pt_vaddr);
Chris Wilsoncc797142013-12-31 15:50:30 +0000977 pt_vaddr = NULL;
Daniel Vettera15326a2013-03-19 23:48:39 +0100978 act_pt++;
Imre Deak6e995e22013-02-18 19:28:04 +0200979 act_pte = 0;
Daniel Vetterdef886c2013-01-24 14:44:56 -0800980 }
Daniel Vetterdef886c2013-01-24 14:44:56 -0800981 }
Chris Wilsoncc797142013-12-31 15:50:30 +0000982 if (pt_vaddr)
983 kunmap_atomic(pt_vaddr);
Daniel Vetterdef886c2013-01-24 14:44:56 -0800984}
985
Ben Widawskya00d8252014-02-19 22:05:48 -0800986static void gen6_ppgtt_unmap_pages(struct i915_hw_ppgtt *ppgtt)
Daniel Vetter1d2a3142012-02-09 17:15:46 +0100987{
Daniel Vetter3440d262013-01-24 13:49:56 -0800988 int i;
989
990 if (ppgtt->pt_dma_addr) {
991 for (i = 0; i < ppgtt->num_pd_entries; i++)
Ben Widawsky853ba5d2013-07-16 16:50:05 -0700992 pci_unmap_page(ppgtt->base.dev->pdev,
Daniel Vetter3440d262013-01-24 13:49:56 -0800993 ppgtt->pt_dma_addr[i],
994 4096, PCI_DMA_BIDIRECTIONAL);
995 }
Ben Widawskya00d8252014-02-19 22:05:48 -0800996}
997
998static void gen6_ppgtt_free(struct i915_hw_ppgtt *ppgtt)
999{
1000 int i;
Daniel Vetter3440d262013-01-24 13:49:56 -08001001
1002 kfree(ppgtt->pt_dma_addr);
1003 for (i = 0; i < ppgtt->num_pd_entries; i++)
1004 __free_page(ppgtt->pt_pages[i]);
1005 kfree(ppgtt->pt_pages);
Daniel Vetter3440d262013-01-24 13:49:56 -08001006}
1007
Ben Widawskya00d8252014-02-19 22:05:48 -08001008static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
1009{
1010 struct i915_hw_ppgtt *ppgtt =
1011 container_of(vm, struct i915_hw_ppgtt, base);
1012
1013 list_del(&vm->global_link);
1014 drm_mm_takedown(&ppgtt->base.mm);
1015 drm_mm_remove_node(&ppgtt->node);
1016
1017 gen6_ppgtt_unmap_pages(ppgtt);
1018 gen6_ppgtt_free(ppgtt);
1019}
1020
Ben Widawskyb1465202014-02-19 22:05:49 -08001021static int gen6_ppgtt_allocate_page_directories(struct i915_hw_ppgtt *ppgtt)
Daniel Vetter3440d262013-01-24 13:49:56 -08001022{
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001023 struct drm_device *dev = ppgtt->base.dev;
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001024 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawskye3cc1992013-12-06 14:11:08 -08001025 bool retried = false;
Ben Widawskyb1465202014-02-19 22:05:49 -08001026 int ret;
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001027
Ben Widawskyc8d4c0d2013-12-06 14:11:07 -08001028 /* PPGTT PDEs reside in the GGTT and consists of 512 entries. The
1029 * allocator works in address space sizes, so it's multiplied by page
1030 * size. We allocate at the top of the GTT to avoid fragmentation.
1031 */
1032 BUG_ON(!drm_mm_initialized(&dev_priv->gtt.base.mm));
Ben Widawskye3cc1992013-12-06 14:11:08 -08001033alloc:
Ben Widawskyc8d4c0d2013-12-06 14:11:07 -08001034 ret = drm_mm_insert_node_in_range_generic(&dev_priv->gtt.base.mm,
1035 &ppgtt->node, GEN6_PD_SIZE,
1036 GEN6_PD_ALIGN, 0,
1037 0, dev_priv->gtt.base.total,
Ben Widawsky3e8b5ae2014-05-06 22:21:30 -07001038 DRM_MM_TOPDOWN);
Ben Widawskye3cc1992013-12-06 14:11:08 -08001039 if (ret == -ENOSPC && !retried) {
1040 ret = i915_gem_evict_something(dev, &dev_priv->gtt.base,
1041 GEN6_PD_SIZE, GEN6_PD_ALIGN,
Daniel Vetterd47c3ea2014-02-14 14:01:18 +01001042 I915_CACHE_NONE, 0);
Ben Widawskye3cc1992013-12-06 14:11:08 -08001043 if (ret)
1044 return ret;
1045
1046 retried = true;
1047 goto alloc;
1048 }
Ben Widawskyc8d4c0d2013-12-06 14:11:07 -08001049
1050 if (ppgtt->node.start < dev_priv->gtt.mappable_end)
1051 DRM_DEBUG("Forced to use aperture for PDEs\n");
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001052
Ben Widawsky6670a5a2013-06-27 16:30:04 -07001053 ppgtt->num_pd_entries = GEN6_PPGTT_PD_ENTRIES;
Ben Widawskyb1465202014-02-19 22:05:49 -08001054 return ret;
1055}
1056
1057static int gen6_ppgtt_allocate_page_tables(struct i915_hw_ppgtt *ppgtt)
1058{
1059 int i;
1060
1061 ppgtt->pt_pages = kcalloc(ppgtt->num_pd_entries, sizeof(struct page *),
1062 GFP_KERNEL);
1063
1064 if (!ppgtt->pt_pages)
1065 return -ENOMEM;
1066
1067 for (i = 0; i < ppgtt->num_pd_entries; i++) {
1068 ppgtt->pt_pages[i] = alloc_page(GFP_KERNEL);
1069 if (!ppgtt->pt_pages[i]) {
1070 gen6_ppgtt_free(ppgtt);
1071 return -ENOMEM;
1072 }
1073 }
1074
1075 return 0;
1076}
1077
1078static int gen6_ppgtt_alloc(struct i915_hw_ppgtt *ppgtt)
1079{
1080 int ret;
1081
1082 ret = gen6_ppgtt_allocate_page_directories(ppgtt);
1083 if (ret)
1084 return ret;
1085
1086 ret = gen6_ppgtt_allocate_page_tables(ppgtt);
1087 if (ret) {
1088 drm_mm_remove_node(&ppgtt->node);
1089 return ret;
1090 }
1091
1092 ppgtt->pt_dma_addr = kcalloc(ppgtt->num_pd_entries, sizeof(dma_addr_t),
1093 GFP_KERNEL);
1094 if (!ppgtt->pt_dma_addr) {
1095 drm_mm_remove_node(&ppgtt->node);
1096 gen6_ppgtt_free(ppgtt);
1097 return -ENOMEM;
1098 }
1099
1100 return 0;
1101}
1102
1103static int gen6_ppgtt_setup_page_tables(struct i915_hw_ppgtt *ppgtt)
1104{
1105 struct drm_device *dev = ppgtt->base.dev;
1106 int i;
1107
1108 for (i = 0; i < ppgtt->num_pd_entries; i++) {
1109 dma_addr_t pt_addr;
1110
1111 pt_addr = pci_map_page(dev->pdev, ppgtt->pt_pages[i], 0, 4096,
1112 PCI_DMA_BIDIRECTIONAL);
1113
1114 if (pci_dma_mapping_error(dev->pdev, pt_addr)) {
1115 gen6_ppgtt_unmap_pages(ppgtt);
1116 return -EIO;
1117 }
1118
1119 ppgtt->pt_dma_addr[i] = pt_addr;
1120 }
1121
1122 return 0;
1123}
1124
1125static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
1126{
1127 struct drm_device *dev = ppgtt->base.dev;
1128 struct drm_i915_private *dev_priv = dev->dev_private;
1129 int ret;
1130
1131 ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
Ben Widawsky48a10382013-12-06 14:11:11 -08001132 if (IS_GEN6(dev)) {
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001133 ppgtt->enable = gen6_ppgtt_enable;
Ben Widawsky48a10382013-12-06 14:11:11 -08001134 ppgtt->switch_mm = gen6_mm_switch;
Ben Widawsky90252e52013-12-06 14:11:12 -08001135 } else if (IS_HASWELL(dev)) {
1136 ppgtt->enable = gen7_ppgtt_enable;
1137 ppgtt->switch_mm = hsw_mm_switch;
Ben Widawsky48a10382013-12-06 14:11:11 -08001138 } else if (IS_GEN7(dev)) {
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001139 ppgtt->enable = gen7_ppgtt_enable;
Ben Widawsky48a10382013-12-06 14:11:11 -08001140 ppgtt->switch_mm = gen7_mm_switch;
1141 } else
Ben Widawskyb4a74e32013-12-06 14:11:09 -08001142 BUG();
Ben Widawskyb1465202014-02-19 22:05:49 -08001143
1144 ret = gen6_ppgtt_alloc(ppgtt);
1145 if (ret)
1146 return ret;
1147
1148 ret = gen6_ppgtt_setup_page_tables(ppgtt);
1149 if (ret) {
1150 gen6_ppgtt_free(ppgtt);
1151 return ret;
1152 }
1153
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001154 ppgtt->base.clear_range = gen6_ppgtt_clear_range;
1155 ppgtt->base.insert_entries = gen6_ppgtt_insert_entries;
1156 ppgtt->base.cleanup = gen6_ppgtt_cleanup;
Ben Widawsky686e1f62013-11-25 09:54:34 -08001157 ppgtt->base.start = 0;
Ben Widawsky5a6c93f2014-03-08 11:58:17 -08001158 ppgtt->base.total = ppgtt->num_pd_entries * I915_PPGTT_PT_ENTRIES * PAGE_SIZE;
Ben Widawskyb1465202014-02-19 22:05:49 -08001159 ppgtt->debug_dump = gen6_dump_ppgtt;
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001160
Ben Widawskyb1465202014-02-19 22:05:49 -08001161 ppgtt->pd_offset =
1162 ppgtt->node.start / PAGE_SIZE * sizeof(gen6_gtt_pte_t);
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001163
Ben Widawsky782f1492014-02-20 11:50:33 -08001164 ppgtt->base.clear_range(&ppgtt->base, 0, ppgtt->base.total, true);
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001165
Ben Widawskyc8d4c0d2013-12-06 14:11:07 -08001166 DRM_DEBUG_DRIVER("Allocated pde space (%ldM) at GTT entry: %lx\n",
1167 ppgtt->node.size >> 20,
1168 ppgtt->node.start / PAGE_SIZE);
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001169
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001170 return 0;
Daniel Vetter3440d262013-01-24 13:49:56 -08001171}
1172
Ben Widawsky246cbfb2013-12-06 14:11:14 -08001173int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
Daniel Vetter3440d262013-01-24 13:49:56 -08001174{
1175 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawskyd6660ad2013-12-06 14:11:13 -08001176 int ret = 0;
Daniel Vetter3440d262013-01-24 13:49:56 -08001177
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001178 ppgtt->base.dev = dev;
Ben Widawsky8407bb92014-03-08 11:58:16 -08001179 ppgtt->base.scratch = dev_priv->gtt.base.scratch;
Daniel Vetter3440d262013-01-24 13:49:56 -08001180
Ben Widawsky3ed124b2013-04-08 18:43:53 -07001181 if (INTEL_INFO(dev)->gen < 8)
1182 ret = gen6_ppgtt_init(ppgtt);
Daniel Vetter8fe6bd22013-11-02 21:07:01 -07001183 else if (IS_GEN8(dev))
Ben Widawsky37aca442013-11-04 20:47:32 -08001184 ret = gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
Ben Widawsky3ed124b2013-04-08 18:43:53 -07001185 else
1186 BUG();
1187
Ben Widawskyc7c48df2013-12-06 14:11:15 -08001188 if (!ret) {
Ben Widawsky7e0d96b2013-12-06 14:11:26 -08001189 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawskyc7c48df2013-12-06 14:11:15 -08001190 kref_init(&ppgtt->ref);
Ben Widawsky93bd8642013-07-16 16:50:06 -07001191 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
1192 ppgtt->base.total);
Ben Widawsky7e0d96b2013-12-06 14:11:26 -08001193 i915_init_vm(dev_priv, &ppgtt->base);
1194 if (INTEL_INFO(dev)->gen < 8) {
Ben Widawsky9f273d42013-12-06 14:11:16 -08001195 gen6_write_pdes(ppgtt);
Ben Widawsky7e0d96b2013-12-06 14:11:26 -08001196 DRM_DEBUG("Adding PPGTT at offset %x\n",
1197 ppgtt->pd_offset << 10);
1198 }
Ben Widawsky93bd8642013-07-16 16:50:06 -07001199 }
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001200
1201 return ret;
1202}
1203
Ben Widawsky7e0d96b2013-12-06 14:11:26 -08001204static void
Ben Widawsky6f65e292013-12-06 14:10:56 -08001205ppgtt_bind_vma(struct i915_vma *vma,
1206 enum i915_cache_level cache_level,
1207 u32 flags)
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001208{
Ben Widawsky782f1492014-02-20 11:50:33 -08001209 vma->vm->insert_entries(vma->vm, vma->obj->pages, vma->node.start,
1210 cache_level);
Daniel Vetter1d2a3142012-02-09 17:15:46 +01001211}
1212
Ben Widawsky7e0d96b2013-12-06 14:11:26 -08001213static void ppgtt_unbind_vma(struct i915_vma *vma)
Daniel Vetter7bddb012012-02-09 17:15:47 +01001214{
Ben Widawsky6f65e292013-12-06 14:10:56 -08001215 vma->vm->clear_range(vma->vm,
Ben Widawsky782f1492014-02-20 11:50:33 -08001216 vma->node.start,
1217 vma->obj->base.size,
Ben Widawsky6f65e292013-12-06 14:10:56 -08001218 true);
Daniel Vetter7bddb012012-02-09 17:15:47 +01001219}
1220
Ben Widawskya81cc002013-01-18 12:30:31 -08001221extern int intel_iommu_gfx_mapped;
1222/* Certain Gen5 chipsets require require idling the GPU before
1223 * unmapping anything from the GTT when VT-d is enabled.
1224 */
1225static inline bool needs_idle_maps(struct drm_device *dev)
1226{
1227#ifdef CONFIG_INTEL_IOMMU
1228 /* Query intel_iommu to see if we need the workaround. Presumably that
1229 * was loaded first.
1230 */
1231 if (IS_GEN5(dev) && IS_MOBILE(dev) && intel_iommu_gfx_mapped)
1232 return true;
1233#endif
1234 return false;
1235}
1236
Ben Widawsky5c042282011-10-17 15:51:55 -07001237static bool do_idling(struct drm_i915_private *dev_priv)
1238{
1239 bool ret = dev_priv->mm.interruptible;
1240
Ben Widawskya81cc002013-01-18 12:30:31 -08001241 if (unlikely(dev_priv->gtt.do_idle_maps)) {
Ben Widawsky5c042282011-10-17 15:51:55 -07001242 dev_priv->mm.interruptible = false;
Ben Widawskyb2da9fe2012-04-26 16:02:58 -07001243 if (i915_gpu_idle(dev_priv->dev)) {
Ben Widawsky5c042282011-10-17 15:51:55 -07001244 DRM_ERROR("Couldn't idle GPU\n");
1245 /* Wait a bit, in hopes it avoids the hang */
1246 udelay(10);
1247 }
1248 }
1249
1250 return ret;
1251}
1252
1253static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
1254{
Ben Widawskya81cc002013-01-18 12:30:31 -08001255 if (unlikely(dev_priv->gtt.do_idle_maps))
Ben Widawsky5c042282011-10-17 15:51:55 -07001256 dev_priv->mm.interruptible = interruptible;
1257}
1258
Ben Widawsky828c7902013-10-16 09:21:30 -07001259void i915_check_and_clear_faults(struct drm_device *dev)
1260{
1261 struct drm_i915_private *dev_priv = dev->dev_private;
Oscar Mateoa4872ba2014-05-22 14:13:33 +01001262 struct intel_engine_cs *ring;
Ben Widawsky828c7902013-10-16 09:21:30 -07001263 int i;
1264
1265 if (INTEL_INFO(dev)->gen < 6)
1266 return;
1267
1268 for_each_ring(ring, dev_priv, i) {
1269 u32 fault_reg;
1270 fault_reg = I915_READ(RING_FAULT_REG(ring));
1271 if (fault_reg & RING_FAULT_VALID) {
1272 DRM_DEBUG_DRIVER("Unexpected fault\n"
1273 "\tAddr: 0x%08lx\\n"
1274 "\tAddress space: %s\n"
1275 "\tSource ID: %d\n"
1276 "\tType: %d\n",
1277 fault_reg & PAGE_MASK,
1278 fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
1279 RING_FAULT_SRCID(fault_reg),
1280 RING_FAULT_FAULT_TYPE(fault_reg));
1281 I915_WRITE(RING_FAULT_REG(ring),
1282 fault_reg & ~RING_FAULT_VALID);
1283 }
1284 }
1285 POSTING_READ(RING_FAULT_REG(&dev_priv->ring[RCS]));
1286}
1287
1288void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
1289{
1290 struct drm_i915_private *dev_priv = dev->dev_private;
1291
1292 /* Don't bother messing with faults pre GEN6 as we have little
1293 * documentation supporting that it's a good idea.
1294 */
1295 if (INTEL_INFO(dev)->gen < 6)
1296 return;
1297
1298 i915_check_and_clear_faults(dev);
1299
1300 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
Ben Widawsky782f1492014-02-20 11:50:33 -08001301 dev_priv->gtt.base.start,
1302 dev_priv->gtt.base.total,
Daniel Vettere568af12014-03-26 20:08:20 +01001303 true);
Ben Widawsky828c7902013-10-16 09:21:30 -07001304}
1305
Daniel Vetter76aaf222010-11-05 22:23:30 +01001306void i915_gem_restore_gtt_mappings(struct drm_device *dev)
1307{
1308 struct drm_i915_private *dev_priv = dev->dev_private;
Chris Wilson05394f32010-11-08 19:18:58 +00001309 struct drm_i915_gem_object *obj;
Ben Widawsky80da2162013-12-06 14:11:17 -08001310 struct i915_address_space *vm;
Daniel Vetter76aaf222010-11-05 22:23:30 +01001311
Ben Widawsky828c7902013-10-16 09:21:30 -07001312 i915_check_and_clear_faults(dev);
1313
Chris Wilsonbee4a182011-01-21 10:54:32 +00001314 /* First fill our portion of the GTT with scratch pages */
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001315 dev_priv->gtt.base.clear_range(&dev_priv->gtt.base,
Ben Widawsky782f1492014-02-20 11:50:33 -08001316 dev_priv->gtt.base.start,
1317 dev_priv->gtt.base.total,
Ben Widawsky828c7902013-10-16 09:21:30 -07001318 true);
Chris Wilsonbee4a182011-01-21 10:54:32 +00001319
Ben Widawsky35c20a62013-05-31 11:28:48 -07001320 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
Ben Widawsky6f65e292013-12-06 14:10:56 -08001321 struct i915_vma *vma = i915_gem_obj_to_vma(obj,
1322 &dev_priv->gtt.base);
1323 if (!vma)
1324 continue;
1325
Chris Wilson2c225692013-08-09 12:26:45 +01001326 i915_gem_clflush_object(obj, obj->pin_display);
Ben Widawsky6f65e292013-12-06 14:10:56 -08001327 /* The bind_vma code tries to be smart about tracking mappings.
1328 * Unfortunately above, we've just wiped out the mappings
1329 * without telling our object about it. So we need to fake it.
1330 */
1331 obj->has_global_gtt_mapping = 0;
1332 vma->bind_vma(vma, obj->cache_level, GLOBAL_BIND);
Daniel Vetter76aaf222010-11-05 22:23:30 +01001333 }
1334
Ben Widawsky80da2162013-12-06 14:11:17 -08001335
Ben Widawskya2319c02014-03-18 16:09:37 -07001336 if (INTEL_INFO(dev)->gen >= 8) {
Ville Syrjäläee0ce472014-04-09 13:28:01 +03001337 if (IS_CHERRYVIEW(dev))
1338 chv_setup_private_ppat(dev_priv);
1339 else
1340 bdw_setup_private_ppat(dev_priv);
1341
Ben Widawsky80da2162013-12-06 14:11:17 -08001342 return;
Ben Widawskya2319c02014-03-18 16:09:37 -07001343 }
Ben Widawsky80da2162013-12-06 14:11:17 -08001344
1345 list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
1346 /* TODO: Perhaps it shouldn't be gen6 specific */
1347 if (i915_is_ggtt(vm)) {
1348 if (dev_priv->mm.aliasing_ppgtt)
1349 gen6_write_pdes(dev_priv->mm.aliasing_ppgtt);
1350 continue;
1351 }
1352
1353 gen6_write_pdes(container_of(vm, struct i915_hw_ppgtt, base));
Daniel Vetter76aaf222010-11-05 22:23:30 +01001354 }
1355
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001356 i915_gem_chipset_flush(dev);
Daniel Vetter76aaf222010-11-05 22:23:30 +01001357}
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01001358
Daniel Vetter74163902012-02-15 23:50:21 +01001359int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01001360{
Chris Wilson9da3da62012-06-01 15:20:22 +01001361 if (obj->has_dma_mapping)
Daniel Vetter74163902012-02-15 23:50:21 +01001362 return 0;
Chris Wilson9da3da62012-06-01 15:20:22 +01001363
1364 if (!dma_map_sg(&obj->base.dev->pdev->dev,
1365 obj->pages->sgl, obj->pages->nents,
1366 PCI_DMA_BIDIRECTIONAL))
1367 return -ENOSPC;
1368
1369 return 0;
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01001370}
1371
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001372static inline void gen8_set_pte(void __iomem *addr, gen8_gtt_pte_t pte)
1373{
1374#ifdef writeq
1375 writeq(pte, addr);
1376#else
1377 iowrite32((u32)pte, addr);
1378 iowrite32(pte >> 32, addr + 4);
1379#endif
1380}
1381
1382static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
1383 struct sg_table *st,
Ben Widawsky782f1492014-02-20 11:50:33 -08001384 uint64_t start,
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001385 enum i915_cache_level level)
1386{
1387 struct drm_i915_private *dev_priv = vm->dev->dev_private;
Ben Widawsky782f1492014-02-20 11:50:33 -08001388 unsigned first_entry = start >> PAGE_SHIFT;
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001389 gen8_gtt_pte_t __iomem *gtt_entries =
1390 (gen8_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
1391 int i = 0;
1392 struct sg_page_iter sg_iter;
Ben Widawsky63c42e52014-04-18 18:04:27 -03001393 dma_addr_t addr = 0;
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001394
1395 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
1396 addr = sg_dma_address(sg_iter.sg) +
1397 (sg_iter.sg_pgoffset << PAGE_SHIFT);
1398 gen8_set_pte(&gtt_entries[i],
1399 gen8_pte_encode(addr, level, true));
1400 i++;
1401 }
1402
1403 /*
1404 * XXX: This serves as a posting read to make sure that the PTE has
1405 * actually been updated. There is some concern that even though
1406 * registers and PTEs are within the same BAR that they are potentially
1407 * of NUMA access patterns. Therefore, even with the way we assume
1408 * hardware should work, we must keep this posting read for paranoia.
1409 */
1410 if (i != 0)
1411 WARN_ON(readq(&gtt_entries[i-1])
1412 != gen8_pte_encode(addr, level, true));
1413
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001414 /* This next bit makes the above posting read even more important. We
1415 * want to flush the TLBs only after we're certain all the PTE updates
1416 * have finished.
1417 */
1418 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1419 POSTING_READ(GFX_FLSH_CNTL_GEN6);
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001420}
1421
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001422/*
1423 * Binds an object into the global gtt with the specified cache level. The object
1424 * will be accessible to the GPU via commands whose operands reference offsets
1425 * within the global GTT as well as accessible by the GPU through the GMADR
1426 * mapped BAR (dev_priv->mm.gtt->gtt).
1427 */
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001428static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001429 struct sg_table *st,
Ben Widawsky782f1492014-02-20 11:50:33 -08001430 uint64_t start,
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001431 enum i915_cache_level level)
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001432{
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001433 struct drm_i915_private *dev_priv = vm->dev->dev_private;
Ben Widawsky782f1492014-02-20 11:50:33 -08001434 unsigned first_entry = start >> PAGE_SHIFT;
Ben Widawskye7c2b582013-04-08 18:43:48 -07001435 gen6_gtt_pte_t __iomem *gtt_entries =
1436 (gen6_gtt_pte_t __iomem *)dev_priv->gtt.gsm + first_entry;
Imre Deak6e995e22013-02-18 19:28:04 +02001437 int i = 0;
1438 struct sg_page_iter sg_iter;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001439 dma_addr_t addr;
1440
Imre Deak6e995e22013-02-18 19:28:04 +02001441 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
Imre Deak2db76d72013-03-26 15:14:18 +02001442 addr = sg_page_iter_dma_address(&sg_iter);
Ben Widawskyb35b3802013-10-16 09:18:21 -07001443 iowrite32(vm->pte_encode(addr, level, true), &gtt_entries[i]);
Imre Deak6e995e22013-02-18 19:28:04 +02001444 i++;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001445 }
1446
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001447 /* XXX: This serves as a posting read to make sure that the PTE has
1448 * actually been updated. There is some concern that even though
1449 * registers and PTEs are within the same BAR that they are potentially
1450 * of NUMA access patterns. Therefore, even with the way we assume
1451 * hardware should work, we must keep this posting read for paranoia.
1452 */
1453 if (i != 0)
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001454 WARN_ON(readl(&gtt_entries[i-1]) !=
Ben Widawskyb35b3802013-10-16 09:18:21 -07001455 vm->pte_encode(addr, level, true));
Ben Widawsky0f9b91c2012-11-04 09:21:30 -08001456
1457 /* This next bit makes the above posting read even more important. We
1458 * want to flush the TLBs only after we're certain all the PTE updates
1459 * have finished.
1460 */
1461 I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
1462 POSTING_READ(GFX_FLSH_CNTL_GEN6);
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001463}
1464
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001465static void gen8_ggtt_clear_range(struct i915_address_space *vm,
Ben Widawsky782f1492014-02-20 11:50:33 -08001466 uint64_t start,
1467 uint64_t length,
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001468 bool use_scratch)
1469{
1470 struct drm_i915_private *dev_priv = vm->dev->dev_private;
Ben Widawsky782f1492014-02-20 11:50:33 -08001471 unsigned first_entry = start >> PAGE_SHIFT;
1472 unsigned num_entries = length >> PAGE_SHIFT;
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001473 gen8_gtt_pte_t scratch_pte, __iomem *gtt_base =
1474 (gen8_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
1475 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
1476 int i;
1477
1478 if (WARN(num_entries > max_entries,
1479 "First entry = %d; Num entries = %d (max=%d)\n",
1480 first_entry, num_entries, max_entries))
1481 num_entries = max_entries;
1482
1483 scratch_pte = gen8_pte_encode(vm->scratch.addr,
1484 I915_CACHE_LLC,
1485 use_scratch);
1486 for (i = 0; i < num_entries; i++)
1487 gen8_set_pte(&gtt_base[i], scratch_pte);
1488 readl(gtt_base);
1489}
1490
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001491static void gen6_ggtt_clear_range(struct i915_address_space *vm,
Ben Widawsky782f1492014-02-20 11:50:33 -08001492 uint64_t start,
1493 uint64_t length,
Ben Widawsky828c7902013-10-16 09:21:30 -07001494 bool use_scratch)
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001495{
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001496 struct drm_i915_private *dev_priv = vm->dev->dev_private;
Ben Widawsky782f1492014-02-20 11:50:33 -08001497 unsigned first_entry = start >> PAGE_SHIFT;
1498 unsigned num_entries = length >> PAGE_SHIFT;
Ben Widawskye7c2b582013-04-08 18:43:48 -07001499 gen6_gtt_pte_t scratch_pte, __iomem *gtt_base =
1500 (gen6_gtt_pte_t __iomem *) dev_priv->gtt.gsm + first_entry;
Ben Widawskya54c0c22013-01-24 14:45:00 -08001501 const int max_entries = gtt_total_entries(dev_priv->gtt) - first_entry;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001502 int i;
1503
1504 if (WARN(num_entries > max_entries,
1505 "First entry = %d; Num entries = %d (max=%d)\n",
1506 first_entry, num_entries, max_entries))
1507 num_entries = max_entries;
1508
Ben Widawsky828c7902013-10-16 09:21:30 -07001509 scratch_pte = vm->pte_encode(vm->scratch.addr, I915_CACHE_LLC, use_scratch);
1510
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001511 for (i = 0; i < num_entries; i++)
1512 iowrite32(scratch_pte, &gtt_base[i]);
1513 readl(gtt_base);
1514}
1515
Ben Widawsky6f65e292013-12-06 14:10:56 -08001516
1517static void i915_ggtt_bind_vma(struct i915_vma *vma,
1518 enum i915_cache_level cache_level,
1519 u32 unused)
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001520{
Ben Widawsky6f65e292013-12-06 14:10:56 -08001521 const unsigned long entry = vma->node.start >> PAGE_SHIFT;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001522 unsigned int flags = (cache_level == I915_CACHE_NONE) ?
1523 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
1524
Ben Widawsky6f65e292013-12-06 14:10:56 -08001525 BUG_ON(!i915_is_ggtt(vma->vm));
1526 intel_gtt_insert_sg_entries(vma->obj->pages, entry, flags);
1527 vma->obj->has_global_gtt_mapping = 1;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001528}
1529
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001530static void i915_ggtt_clear_range(struct i915_address_space *vm,
Ben Widawsky782f1492014-02-20 11:50:33 -08001531 uint64_t start,
1532 uint64_t length,
Ben Widawsky828c7902013-10-16 09:21:30 -07001533 bool unused)
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001534{
Ben Widawsky782f1492014-02-20 11:50:33 -08001535 unsigned first_entry = start >> PAGE_SHIFT;
1536 unsigned num_entries = length >> PAGE_SHIFT;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001537 intel_gtt_clear_range(first_entry, num_entries);
1538}
1539
Ben Widawsky6f65e292013-12-06 14:10:56 -08001540static void i915_ggtt_unbind_vma(struct i915_vma *vma)
Chris Wilsond5bd1442011-04-14 06:48:26 +01001541{
Ben Widawsky6f65e292013-12-06 14:10:56 -08001542 const unsigned int first = vma->node.start >> PAGE_SHIFT;
1543 const unsigned int size = vma->obj->base.size >> PAGE_SHIFT;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001544
Ben Widawsky6f65e292013-12-06 14:10:56 -08001545 BUG_ON(!i915_is_ggtt(vma->vm));
1546 vma->obj->has_global_gtt_mapping = 0;
1547 intel_gtt_clear_range(first, size);
Chris Wilsond5bd1442011-04-14 06:48:26 +01001548}
1549
Ben Widawsky6f65e292013-12-06 14:10:56 -08001550static void ggtt_bind_vma(struct i915_vma *vma,
1551 enum i915_cache_level cache_level,
1552 u32 flags)
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01001553{
Ben Widawsky6f65e292013-12-06 14:10:56 -08001554 struct drm_device *dev = vma->vm->dev;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001555 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawsky6f65e292013-12-06 14:10:56 -08001556 struct drm_i915_gem_object *obj = vma->obj;
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08001557
Ben Widawsky6f65e292013-12-06 14:10:56 -08001558 /* If there is no aliasing PPGTT, or the caller needs a global mapping,
1559 * or we have a global mapping already but the cacheability flags have
1560 * changed, set the global PTEs.
1561 *
1562 * If there is an aliasing PPGTT it is anecdotally faster, so use that
1563 * instead if none of the above hold true.
1564 *
1565 * NB: A global mapping should only be needed for special regions like
1566 * "gtt mappable", SNB errata, or if specified via special execbuf
1567 * flags. At all other times, the GPU will use the aliasing PPGTT.
1568 */
1569 if (!dev_priv->mm.aliasing_ppgtt || flags & GLOBAL_BIND) {
1570 if (!obj->has_global_gtt_mapping ||
1571 (cache_level != obj->cache_level)) {
Ben Widawsky782f1492014-02-20 11:50:33 -08001572 vma->vm->insert_entries(vma->vm, obj->pages,
1573 vma->node.start,
Ben Widawsky6f65e292013-12-06 14:10:56 -08001574 cache_level);
1575 obj->has_global_gtt_mapping = 1;
1576 }
1577 }
Daniel Vetter74898d72012-02-15 23:50:22 +01001578
Ben Widawsky6f65e292013-12-06 14:10:56 -08001579 if (dev_priv->mm.aliasing_ppgtt &&
1580 (!obj->has_aliasing_ppgtt_mapping ||
1581 (cache_level != obj->cache_level))) {
1582 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
1583 appgtt->base.insert_entries(&appgtt->base,
Ben Widawsky782f1492014-02-20 11:50:33 -08001584 vma->obj->pages,
1585 vma->node.start,
1586 cache_level);
Ben Widawsky6f65e292013-12-06 14:10:56 -08001587 vma->obj->has_aliasing_ppgtt_mapping = 1;
1588 }
1589}
1590
1591static void ggtt_unbind_vma(struct i915_vma *vma)
1592{
1593 struct drm_device *dev = vma->vm->dev;
1594 struct drm_i915_private *dev_priv = dev->dev_private;
1595 struct drm_i915_gem_object *obj = vma->obj;
Ben Widawsky6f65e292013-12-06 14:10:56 -08001596
1597 if (obj->has_global_gtt_mapping) {
Ben Widawsky782f1492014-02-20 11:50:33 -08001598 vma->vm->clear_range(vma->vm,
1599 vma->node.start,
1600 obj->base.size,
Ben Widawsky6f65e292013-12-06 14:10:56 -08001601 true);
1602 obj->has_global_gtt_mapping = 0;
1603 }
1604
1605 if (obj->has_aliasing_ppgtt_mapping) {
1606 struct i915_hw_ppgtt *appgtt = dev_priv->mm.aliasing_ppgtt;
1607 appgtt->base.clear_range(&appgtt->base,
Ben Widawsky782f1492014-02-20 11:50:33 -08001608 vma->node.start,
1609 obj->base.size,
Ben Widawsky6f65e292013-12-06 14:10:56 -08001610 true);
1611 obj->has_aliasing_ppgtt_mapping = 0;
1612 }
Daniel Vetter74163902012-02-15 23:50:21 +01001613}
1614
1615void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
1616{
Ben Widawsky5c042282011-10-17 15:51:55 -07001617 struct drm_device *dev = obj->base.dev;
1618 struct drm_i915_private *dev_priv = dev->dev_private;
1619 bool interruptible;
1620
1621 interruptible = do_idling(dev_priv);
1622
Chris Wilson9da3da62012-06-01 15:20:22 +01001623 if (!obj->has_dma_mapping)
1624 dma_unmap_sg(&dev->pdev->dev,
1625 obj->pages->sgl, obj->pages->nents,
1626 PCI_DMA_BIDIRECTIONAL);
Ben Widawsky5c042282011-10-17 15:51:55 -07001627
1628 undo_idling(dev_priv, interruptible);
Daniel Vetter7c2e6fd2010-11-06 10:10:47 +01001629}
Daniel Vetter644ec022012-03-26 09:45:40 +02001630
Chris Wilson42d6ab42012-07-26 11:49:32 +01001631static void i915_gtt_color_adjust(struct drm_mm_node *node,
1632 unsigned long color,
1633 unsigned long *start,
1634 unsigned long *end)
1635{
1636 if (node->color != color)
1637 *start += 4096;
1638
1639 if (!list_empty(&node->node_list)) {
1640 node = list_entry(node->node_list.next,
1641 struct drm_mm_node,
1642 node_list);
1643 if (node->allocated && node->color != color)
1644 *end -= 4096;
1645 }
1646}
Ben Widawskyfbe5d362013-11-04 19:56:49 -08001647
Ben Widawskyd7e50082012-12-18 10:31:25 -08001648void i915_gem_setup_global_gtt(struct drm_device *dev,
1649 unsigned long start,
1650 unsigned long mappable_end,
1651 unsigned long end)
Daniel Vetter644ec022012-03-26 09:45:40 +02001652{
Ben Widawskye78891c2013-01-25 16:41:04 -08001653 /* Let GEM Manage all of the aperture.
1654 *
1655 * However, leave one page at the end still bound to the scratch page.
1656 * There are a number of places where the hardware apparently prefetches
1657 * past the end of the object, and we've seen multiple hangs with the
1658 * GPU head pointer stuck in a batchbuffer bound at the last page of the
1659 * aperture. One page should be enough to keep any prefetching inside
1660 * of the aperture.
1661 */
Ben Widawsky40d749802013-07-31 16:59:59 -07001662 struct drm_i915_private *dev_priv = dev->dev_private;
1663 struct i915_address_space *ggtt_vm = &dev_priv->gtt.base;
Chris Wilsoned2f3452012-11-15 11:32:19 +00001664 struct drm_mm_node *entry;
1665 struct drm_i915_gem_object *obj;
1666 unsigned long hole_start, hole_end;
Daniel Vetter644ec022012-03-26 09:45:40 +02001667
Ben Widawsky35451cb2013-01-17 12:45:13 -08001668 BUG_ON(mappable_end > end);
1669
Chris Wilsoned2f3452012-11-15 11:32:19 +00001670 /* Subtract the guard page ... */
Ben Widawsky40d749802013-07-31 16:59:59 -07001671 drm_mm_init(&ggtt_vm->mm, start, end - start - PAGE_SIZE);
Chris Wilson42d6ab42012-07-26 11:49:32 +01001672 if (!HAS_LLC(dev))
Ben Widawsky93bd8642013-07-16 16:50:06 -07001673 dev_priv->gtt.base.mm.color_adjust = i915_gtt_color_adjust;
Daniel Vetter644ec022012-03-26 09:45:40 +02001674
Chris Wilsoned2f3452012-11-15 11:32:19 +00001675 /* Mark any preallocated objects as occupied */
Ben Widawsky35c20a62013-05-31 11:28:48 -07001676 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
Ben Widawsky40d749802013-07-31 16:59:59 -07001677 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
Ben Widawskyb3a070c2013-07-05 14:41:02 -07001678 int ret;
Ben Widawskyedd41a82013-07-05 14:41:05 -07001679 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
Ben Widawskyc6cfb322013-07-05 14:41:06 -07001680 i915_gem_obj_ggtt_offset(obj), obj->base.size);
Chris Wilsoned2f3452012-11-15 11:32:19 +00001681
Ben Widawskyc6cfb322013-07-05 14:41:06 -07001682 WARN_ON(i915_gem_obj_ggtt_bound(obj));
Ben Widawsky40d749802013-07-31 16:59:59 -07001683 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
Ben Widawskyc6cfb322013-07-05 14:41:06 -07001684 if (ret)
Ben Widawskyb3a070c2013-07-05 14:41:02 -07001685 DRM_DEBUG_KMS("Reservation failed\n");
Chris Wilsoned2f3452012-11-15 11:32:19 +00001686 obj->has_global_gtt_mapping = 1;
1687 }
1688
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001689 dev_priv->gtt.base.start = start;
1690 dev_priv->gtt.base.total = end - start;
Daniel Vetter644ec022012-03-26 09:45:40 +02001691
Chris Wilsoned2f3452012-11-15 11:32:19 +00001692 /* Clear any non-preallocated blocks */
Ben Widawsky40d749802013-07-31 16:59:59 -07001693 drm_mm_for_each_hole(entry, &ggtt_vm->mm, hole_start, hole_end) {
Chris Wilsoned2f3452012-11-15 11:32:19 +00001694 DRM_DEBUG_KMS("clearing unused GTT space: [%lx, %lx]\n",
1695 hole_start, hole_end);
Ben Widawsky782f1492014-02-20 11:50:33 -08001696 ggtt_vm->clear_range(ggtt_vm, hole_start,
1697 hole_end - hole_start, true);
Chris Wilsoned2f3452012-11-15 11:32:19 +00001698 }
1699
1700 /* And finally clear the reserved guard page */
Ben Widawsky782f1492014-02-20 11:50:33 -08001701 ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001702}
1703
Ben Widawskyd7e50082012-12-18 10:31:25 -08001704void i915_gem_init_global_gtt(struct drm_device *dev)
1705{
1706 struct drm_i915_private *dev_priv = dev->dev_private;
1707 unsigned long gtt_size, mappable_size;
Ben Widawskyd7e50082012-12-18 10:31:25 -08001708
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001709 gtt_size = dev_priv->gtt.base.total;
Ben Widawsky93d18792013-01-17 12:45:17 -08001710 mappable_size = dev_priv->gtt.mappable_end;
Ben Widawskyd7e50082012-12-18 10:31:25 -08001711
Ben Widawskye78891c2013-01-25 16:41:04 -08001712 i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001713}
1714
1715static int setup_scratch_page(struct drm_device *dev)
1716{
1717 struct drm_i915_private *dev_priv = dev->dev_private;
1718 struct page *page;
1719 dma_addr_t dma_addr;
1720
1721 page = alloc_page(GFP_KERNEL | GFP_DMA32 | __GFP_ZERO);
1722 if (page == NULL)
1723 return -ENOMEM;
1724 get_page(page);
1725 set_pages_uc(page, 1);
1726
1727#ifdef CONFIG_INTEL_IOMMU
1728 dma_addr = pci_map_page(dev->pdev, page, 0, PAGE_SIZE,
1729 PCI_DMA_BIDIRECTIONAL);
1730 if (pci_dma_mapping_error(dev->pdev, dma_addr))
1731 return -EINVAL;
1732#else
1733 dma_addr = page_to_phys(page);
1734#endif
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001735 dev_priv->gtt.base.scratch.page = page;
1736 dev_priv->gtt.base.scratch.addr = dma_addr;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001737
1738 return 0;
1739}
1740
1741static void teardown_scratch_page(struct drm_device *dev)
1742{
1743 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001744 struct page *page = dev_priv->gtt.base.scratch.page;
1745
1746 set_pages_wb(page, 1);
1747 pci_unmap_page(dev->pdev, dev_priv->gtt.base.scratch.addr,
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001748 PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001749 put_page(page);
1750 __free_page(page);
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001751}
1752
1753static inline unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl)
1754{
1755 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT;
1756 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK;
1757 return snb_gmch_ctl << 20;
1758}
1759
Ben Widawsky9459d252013-11-03 16:53:55 -08001760static inline unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl)
1761{
1762 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT;
1763 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK;
1764 if (bdw_gmch_ctl)
1765 bdw_gmch_ctl = 1 << bdw_gmch_ctl;
1766 return bdw_gmch_ctl << 20;
1767}
1768
Damien Lespiaud7f25f22014-05-08 22:19:40 +03001769static inline unsigned int chv_get_total_gtt_size(u16 gmch_ctrl)
1770{
1771 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT;
1772 gmch_ctrl &= SNB_GMCH_GGMS_MASK;
1773
1774 if (gmch_ctrl)
1775 return 1 << (20 + gmch_ctrl);
1776
1777 return 0;
1778}
1779
Ben Widawskybaa09f52013-01-24 13:49:57 -08001780static inline size_t gen6_get_stolen_size(u16 snb_gmch_ctl)
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001781{
1782 snb_gmch_ctl >>= SNB_GMCH_GMS_SHIFT;
1783 snb_gmch_ctl &= SNB_GMCH_GMS_MASK;
1784 return snb_gmch_ctl << 25; /* 32 MB units */
1785}
1786
Ben Widawsky9459d252013-11-03 16:53:55 -08001787static inline size_t gen8_get_stolen_size(u16 bdw_gmch_ctl)
1788{
1789 bdw_gmch_ctl >>= BDW_GMCH_GMS_SHIFT;
1790 bdw_gmch_ctl &= BDW_GMCH_GMS_MASK;
1791 return bdw_gmch_ctl << 25; /* 32 MB units */
1792}
1793
Damien Lespiaud7f25f22014-05-08 22:19:40 +03001794static size_t chv_get_stolen_size(u16 gmch_ctrl)
1795{
1796 gmch_ctrl >>= SNB_GMCH_GMS_SHIFT;
1797 gmch_ctrl &= SNB_GMCH_GMS_MASK;
1798
1799 /*
1800 * 0x0 to 0x10: 32MB increments starting at 0MB
1801 * 0x11 to 0x16: 4MB increments starting at 8MB
1802 * 0x17 to 0x1d: 4MB increments start at 36MB
1803 */
1804 if (gmch_ctrl < 0x11)
1805 return gmch_ctrl << 25;
1806 else if (gmch_ctrl < 0x17)
1807 return (gmch_ctrl - 0x11 + 2) << 22;
1808 else
1809 return (gmch_ctrl - 0x17 + 9) << 22;
1810}
1811
Ben Widawsky63340132013-11-04 19:32:22 -08001812static int ggtt_probe_common(struct drm_device *dev,
1813 size_t gtt_size)
1814{
1815 struct drm_i915_private *dev_priv = dev->dev_private;
Bjorn Helgaas21c34602013-12-21 10:52:52 -07001816 phys_addr_t gtt_phys_addr;
Ben Widawsky63340132013-11-04 19:32:22 -08001817 int ret;
1818
1819 /* For Modern GENs the PTEs and register space are split in the BAR */
Bjorn Helgaas21c34602013-12-21 10:52:52 -07001820 gtt_phys_addr = pci_resource_start(dev->pdev, 0) +
Ben Widawsky63340132013-11-04 19:32:22 -08001821 (pci_resource_len(dev->pdev, 0) / 2);
1822
Bjorn Helgaas21c34602013-12-21 10:52:52 -07001823 dev_priv->gtt.gsm = ioremap_wc(gtt_phys_addr, gtt_size);
Ben Widawsky63340132013-11-04 19:32:22 -08001824 if (!dev_priv->gtt.gsm) {
1825 DRM_ERROR("Failed to map the gtt page table\n");
1826 return -ENOMEM;
1827 }
1828
1829 ret = setup_scratch_page(dev);
1830 if (ret) {
1831 DRM_ERROR("Scratch setup failed\n");
1832 /* iounmap will also get called at remove, but meh */
1833 iounmap(dev_priv->gtt.gsm);
1834 }
1835
1836 return ret;
1837}
1838
Ben Widawskyfbe5d362013-11-04 19:56:49 -08001839/* The GGTT and PPGTT need a private PPAT setup in order to handle cacheability
1840 * bits. When using advanced contexts each context stores its own PAT, but
1841 * writing this data shouldn't be harmful even in those cases. */
Ville Syrjäläee0ce472014-04-09 13:28:01 +03001842static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv)
Ben Widawskyfbe5d362013-11-04 19:56:49 -08001843{
Ben Widawskyfbe5d362013-11-04 19:56:49 -08001844 uint64_t pat;
1845
1846 pat = GEN8_PPAT(0, GEN8_PPAT_WB | GEN8_PPAT_LLC) | /* for normal objects, no eLLC */
1847 GEN8_PPAT(1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC) | /* for something pointing to ptes? */
1848 GEN8_PPAT(2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC) | /* for scanout with eLLC */
1849 GEN8_PPAT(3, GEN8_PPAT_UC) | /* Uncached objects, mostly for scanout */
1850 GEN8_PPAT(4, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(0)) |
1851 GEN8_PPAT(5, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(1)) |
1852 GEN8_PPAT(6, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(2)) |
1853 GEN8_PPAT(7, GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3));
1854
1855 /* XXX: spec defines this as 2 distinct registers. It's unclear if a 64b
1856 * write would work. */
1857 I915_WRITE(GEN8_PRIVATE_PAT, pat);
1858 I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
1859}
1860
Ville Syrjäläee0ce472014-04-09 13:28:01 +03001861static void chv_setup_private_ppat(struct drm_i915_private *dev_priv)
1862{
1863 uint64_t pat;
1864
1865 /*
1866 * Map WB on BDW to snooped on CHV.
1867 *
1868 * Only the snoop bit has meaning for CHV, the rest is
1869 * ignored.
1870 *
1871 * Note that the harware enforces snooping for all page
1872 * table accesses. The snoop bit is actually ignored for
1873 * PDEs.
1874 */
1875 pat = GEN8_PPAT(0, CHV_PPAT_SNOOP) |
1876 GEN8_PPAT(1, 0) |
1877 GEN8_PPAT(2, 0) |
1878 GEN8_PPAT(3, 0) |
1879 GEN8_PPAT(4, CHV_PPAT_SNOOP) |
1880 GEN8_PPAT(5, CHV_PPAT_SNOOP) |
1881 GEN8_PPAT(6, CHV_PPAT_SNOOP) |
1882 GEN8_PPAT(7, CHV_PPAT_SNOOP);
1883
1884 I915_WRITE(GEN8_PRIVATE_PAT, pat);
1885 I915_WRITE(GEN8_PRIVATE_PAT + 4, pat >> 32);
1886}
1887
Ben Widawsky63340132013-11-04 19:32:22 -08001888static int gen8_gmch_probe(struct drm_device *dev,
1889 size_t *gtt_total,
1890 size_t *stolen,
1891 phys_addr_t *mappable_base,
1892 unsigned long *mappable_end)
1893{
1894 struct drm_i915_private *dev_priv = dev->dev_private;
1895 unsigned int gtt_size;
1896 u16 snb_gmch_ctl;
1897 int ret;
1898
1899 /* TODO: We're not aware of mappable constraints on gen8 yet */
1900 *mappable_base = pci_resource_start(dev->pdev, 2);
1901 *mappable_end = pci_resource_len(dev->pdev, 2);
1902
1903 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(39)))
1904 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(39));
1905
1906 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
1907
Damien Lespiaud7f25f22014-05-08 22:19:40 +03001908 if (IS_CHERRYVIEW(dev)) {
1909 *stolen = chv_get_stolen_size(snb_gmch_ctl);
1910 gtt_size = chv_get_total_gtt_size(snb_gmch_ctl);
1911 } else {
1912 *stolen = gen8_get_stolen_size(snb_gmch_ctl);
1913 gtt_size = gen8_get_total_gtt_size(snb_gmch_ctl);
1914 }
Ben Widawsky63340132013-11-04 19:32:22 -08001915
Ben Widawskyd31eb102013-11-02 21:07:17 -07001916 *gtt_total = (gtt_size / sizeof(gen8_gtt_pte_t)) << PAGE_SHIFT;
Ben Widawsky63340132013-11-04 19:32:22 -08001917
Ville Syrjäläee0ce472014-04-09 13:28:01 +03001918 if (IS_CHERRYVIEW(dev))
1919 chv_setup_private_ppat(dev_priv);
1920 else
1921 bdw_setup_private_ppat(dev_priv);
Ben Widawskyfbe5d362013-11-04 19:56:49 -08001922
Ben Widawsky63340132013-11-04 19:32:22 -08001923 ret = ggtt_probe_common(dev, gtt_size);
1924
Ben Widawsky94ec8f62013-11-02 21:07:18 -07001925 dev_priv->gtt.base.clear_range = gen8_ggtt_clear_range;
1926 dev_priv->gtt.base.insert_entries = gen8_ggtt_insert_entries;
Ben Widawsky63340132013-11-04 19:32:22 -08001927
1928 return ret;
1929}
1930
Ben Widawskybaa09f52013-01-24 13:49:57 -08001931static int gen6_gmch_probe(struct drm_device *dev,
1932 size_t *gtt_total,
Ben Widawsky41907dd2013-02-08 11:32:47 -08001933 size_t *stolen,
1934 phys_addr_t *mappable_base,
1935 unsigned long *mappable_end)
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001936{
1937 struct drm_i915_private *dev_priv = dev->dev_private;
Ben Widawskybaa09f52013-01-24 13:49:57 -08001938 unsigned int gtt_size;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001939 u16 snb_gmch_ctl;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001940 int ret;
1941
Ben Widawsky41907dd2013-02-08 11:32:47 -08001942 *mappable_base = pci_resource_start(dev->pdev, 2);
1943 *mappable_end = pci_resource_len(dev->pdev, 2);
1944
Ben Widawskybaa09f52013-01-24 13:49:57 -08001945 /* 64/512MB is the current min/max we actually know of, but this is just
1946 * a coarse sanity check.
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001947 */
Ben Widawsky41907dd2013-02-08 11:32:47 -08001948 if ((*mappable_end < (64<<20) || (*mappable_end > (512<<20)))) {
Ben Widawskybaa09f52013-01-24 13:49:57 -08001949 DRM_ERROR("Unknown GMADR size (%lx)\n",
1950 dev_priv->gtt.mappable_end);
1951 return -ENXIO;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001952 }
1953
Ben Widawskye76e9ae2012-11-04 09:21:27 -08001954 if (!pci_set_dma_mask(dev->pdev, DMA_BIT_MASK(40)))
1955 pci_set_consistent_dma_mask(dev->pdev, DMA_BIT_MASK(40));
Ben Widawskybaa09f52013-01-24 13:49:57 -08001956 pci_read_config_word(dev->pdev, SNB_GMCH_CTRL, &snb_gmch_ctl);
Ben Widawskybaa09f52013-01-24 13:49:57 -08001957
Ben Widawskyc4ae25e2013-05-01 11:00:34 -07001958 *stolen = gen6_get_stolen_size(snb_gmch_ctl);
Ben Widawskybaa09f52013-01-24 13:49:57 -08001959
Ben Widawsky63340132013-11-04 19:32:22 -08001960 gtt_size = gen6_get_total_gtt_size(snb_gmch_ctl);
Ben Widawskybaa09f52013-01-24 13:49:57 -08001961 *gtt_total = (gtt_size / sizeof(gen6_gtt_pte_t)) << PAGE_SHIFT;
1962
Ben Widawsky63340132013-11-04 19:32:22 -08001963 ret = ggtt_probe_common(dev, gtt_size);
Ben Widawskybaa09f52013-01-24 13:49:57 -08001964
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001965 dev_priv->gtt.base.clear_range = gen6_ggtt_clear_range;
1966 dev_priv->gtt.base.insert_entries = gen6_ggtt_insert_entries;
Ben Widawskybaa09f52013-01-24 13:49:57 -08001967
1968 return ret;
1969}
1970
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001971static void gen6_gmch_remove(struct i915_address_space *vm)
Ben Widawskybaa09f52013-01-24 13:49:57 -08001972{
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001973
1974 struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
Ben Widawsky5ed16782013-11-25 09:54:43 -08001975
1976 drm_mm_takedown(&vm->mm);
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001977 iounmap(gtt->gsm);
1978 teardown_scratch_page(vm->dev);
Ben Widawskybaa09f52013-01-24 13:49:57 -08001979}
1980
1981static int i915_gmch_probe(struct drm_device *dev,
1982 size_t *gtt_total,
Ben Widawsky41907dd2013-02-08 11:32:47 -08001983 size_t *stolen,
1984 phys_addr_t *mappable_base,
1985 unsigned long *mappable_end)
Ben Widawskybaa09f52013-01-24 13:49:57 -08001986{
1987 struct drm_i915_private *dev_priv = dev->dev_private;
1988 int ret;
1989
Ben Widawskybaa09f52013-01-24 13:49:57 -08001990 ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
1991 if (!ret) {
1992 DRM_ERROR("failed to set up gmch\n");
1993 return -EIO;
1994 }
1995
Ben Widawsky41907dd2013-02-08 11:32:47 -08001996 intel_gtt_get(gtt_total, stolen, mappable_base, mappable_end);
Ben Widawskybaa09f52013-01-24 13:49:57 -08001997
1998 dev_priv->gtt.do_idle_maps = needs_idle_maps(dev_priv->dev);
Ben Widawsky853ba5d2013-07-16 16:50:05 -07001999 dev_priv->gtt.base.clear_range = i915_ggtt_clear_range;
Ben Widawskybaa09f52013-01-24 13:49:57 -08002000
Chris Wilsonc0a7f812013-12-30 12:16:15 +00002001 if (unlikely(dev_priv->gtt.do_idle_maps))
2002 DRM_INFO("applying Ironlake quirks for intel_iommu\n");
2003
Ben Widawskybaa09f52013-01-24 13:49:57 -08002004 return 0;
2005}
2006
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002007static void i915_gmch_remove(struct i915_address_space *vm)
Ben Widawskybaa09f52013-01-24 13:49:57 -08002008{
2009 intel_gmch_remove();
2010}
2011
2012int i915_gem_gtt_init(struct drm_device *dev)
2013{
2014 struct drm_i915_private *dev_priv = dev->dev_private;
2015 struct i915_gtt *gtt = &dev_priv->gtt;
Ben Widawskybaa09f52013-01-24 13:49:57 -08002016 int ret;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002017
Ben Widawskybaa09f52013-01-24 13:49:57 -08002018 if (INTEL_INFO(dev)->gen <= 5) {
Ben Widawskyb2f21b42013-06-27 16:30:20 -07002019 gtt->gtt_probe = i915_gmch_probe;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002020 gtt->base.cleanup = i915_gmch_remove;
Ben Widawsky63340132013-11-04 19:32:22 -08002021 } else if (INTEL_INFO(dev)->gen < 8) {
Ben Widawskyb2f21b42013-06-27 16:30:20 -07002022 gtt->gtt_probe = gen6_gmch_probe;
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002023 gtt->base.cleanup = gen6_gmch_remove;
Ben Widawsky4d15c142013-07-04 11:02:06 -07002024 if (IS_HASWELL(dev) && dev_priv->ellc_size)
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002025 gtt->base.pte_encode = iris_pte_encode;
Ben Widawsky4d15c142013-07-04 11:02:06 -07002026 else if (IS_HASWELL(dev))
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002027 gtt->base.pte_encode = hsw_pte_encode;
Ben Widawskyb2f21b42013-06-27 16:30:20 -07002028 else if (IS_VALLEYVIEW(dev))
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002029 gtt->base.pte_encode = byt_pte_encode;
Chris Wilson350ec882013-08-06 13:17:02 +01002030 else if (INTEL_INFO(dev)->gen >= 7)
2031 gtt->base.pte_encode = ivb_pte_encode;
Ben Widawskyb2f21b42013-06-27 16:30:20 -07002032 else
Chris Wilson350ec882013-08-06 13:17:02 +01002033 gtt->base.pte_encode = snb_pte_encode;
Ben Widawsky63340132013-11-04 19:32:22 -08002034 } else {
2035 dev_priv->gtt.gtt_probe = gen8_gmch_probe;
2036 dev_priv->gtt.base.cleanup = gen6_gmch_remove;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002037 }
2038
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002039 ret = gtt->gtt_probe(dev, &gtt->base.total, &gtt->stolen_size,
Ben Widawskyb2f21b42013-06-27 16:30:20 -07002040 &gtt->mappable_base, &gtt->mappable_end);
Ben Widawskya54c0c22013-01-24 14:45:00 -08002041 if (ret)
Ben Widawskybaa09f52013-01-24 13:49:57 -08002042 return ret;
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002043
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002044 gtt->base.dev = dev;
2045
Ben Widawskybaa09f52013-01-24 13:49:57 -08002046 /* GMADR is the PCI mmio aperture into the global GTT. */
Ben Widawsky853ba5d2013-07-16 16:50:05 -07002047 DRM_INFO("Memory usable by graphics device = %zdM\n",
2048 gtt->base.total >> 20);
Ben Widawskyb2f21b42013-06-27 16:30:20 -07002049 DRM_DEBUG_DRIVER("GMADR size = %ldM\n", gtt->mappable_end >> 20);
2050 DRM_DEBUG_DRIVER("GTT stolen size = %zdM\n", gtt->stolen_size >> 20);
Daniel Vetter5db6c732014-03-31 16:23:04 +02002051#ifdef CONFIG_INTEL_IOMMU
2052 if (intel_iommu_gfx_mapped)
2053 DRM_INFO("VT-d active for gfx access\n");
2054#endif
Daniel Vetter7faf1ab2013-01-24 14:44:55 -08002055
Ben Widawskye76e9ae2012-11-04 09:21:27 -08002056 return 0;
Daniel Vetter644ec022012-03-26 09:45:40 +02002057}
Ben Widawsky6f65e292013-12-06 14:10:56 -08002058
2059static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
2060 struct i915_address_space *vm)
2061{
2062 struct i915_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL);
2063 if (vma == NULL)
2064 return ERR_PTR(-ENOMEM);
2065
2066 INIT_LIST_HEAD(&vma->vma_link);
2067 INIT_LIST_HEAD(&vma->mm_list);
2068 INIT_LIST_HEAD(&vma->exec_list);
2069 vma->vm = vm;
2070 vma->obj = obj;
2071
2072 switch (INTEL_INFO(vm->dev)->gen) {
2073 case 8:
2074 case 7:
2075 case 6:
Ben Widawsky7e0d96b2013-12-06 14:11:26 -08002076 if (i915_is_ggtt(vm)) {
2077 vma->unbind_vma = ggtt_unbind_vma;
2078 vma->bind_vma = ggtt_bind_vma;
2079 } else {
2080 vma->unbind_vma = ppgtt_unbind_vma;
2081 vma->bind_vma = ppgtt_bind_vma;
2082 }
Ben Widawsky6f65e292013-12-06 14:10:56 -08002083 break;
2084 case 5:
2085 case 4:
2086 case 3:
2087 case 2:
2088 BUG_ON(!i915_is_ggtt(vm));
2089 vma->unbind_vma = i915_ggtt_unbind_vma;
2090 vma->bind_vma = i915_ggtt_bind_vma;
2091 break;
2092 default:
2093 BUG();
2094 }
2095
2096 /* Keep GGTT vmas first to make debug easier */
2097 if (i915_is_ggtt(vm))
2098 list_add(&vma->vma_link, &obj->vma_list);
2099 else
2100 list_add_tail(&vma->vma_link, &obj->vma_list);
2101
2102 return vma;
2103}
2104
2105struct i915_vma *
2106i915_gem_obj_lookup_or_create_vma(struct drm_i915_gem_object *obj,
2107 struct i915_address_space *vm)
2108{
2109 struct i915_vma *vma;
2110
2111 vma = i915_gem_obj_to_vma(obj, vm);
2112 if (!vma)
2113 vma = __i915_gem_vma_create(obj, vm);
2114
2115 return vma;
2116}