blob: 09874d695188067dca03540efd5b2f135a75b913 [file] [log] [blame]
Pauli Nieminen1403b1a2010-04-01 12:44:57 +00001/*
2 * Copyright (c) Red Hat Inc.
3
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sub license,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the
12 * next paragraph) shall be included in all copies or substantial portions
13 * of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 *
23 * Authors: Dave Airlie <airlied@redhat.com>
24 * Jerome Glisse <jglisse@redhat.com>
25 * Pauli Nieminen <suokkos@gmail.com>
26 */
27
28/* simple list based uncached page pool
29 * - Pool collects resently freed pages for reuse
30 * - Use page->lru to keep a free list
31 * - doesn't track currently in use pages
32 */
Joe Perches25d04792012-03-16 21:43:50 -070033
34#define pr_fmt(fmt) "[TTM] " fmt
35
Pauli Nieminen1403b1a2010-04-01 12:44:57 +000036#include <linux/list.h>
37#include <linux/spinlock.h>
38#include <linux/highmem.h>
39#include <linux/mm_types.h>
Pauli Nieminen07458662010-04-01 12:44:58 +000040#include <linux/module.h>
Pauli Nieminen1403b1a2010-04-01 12:44:57 +000041#include <linux/mm.h>
Matt Turner4cdc8402010-04-07 22:42:04 +000042#include <linux/seq_file.h> /* for seq_printf */
Stephen Rothwell2125b8a2010-04-08 13:42:03 +100043#include <linux/slab.h>
Konrad Rzeszutek Wilkf9820a42010-11-29 13:52:18 -050044#include <linux/dma-mapping.h>
Pauli Nieminen1403b1a2010-04-01 12:44:57 +000045
Arun Sharma600634972011-07-26 16:09:06 -070046#include <linux/atomic.h>
Pauli Nieminen1403b1a2010-04-01 12:44:57 +000047
David Howells760285e2012-10-02 18:01:07 +010048#include <drm/ttm/ttm_bo_driver.h>
49#include <drm/ttm/ttm_page_alloc.h>
Pauli Nieminen1403b1a2010-04-01 12:44:57 +000050
Luck, Tonyd6678652010-07-21 10:15:39 -070051#ifdef TTM_HAS_AGP
52#include <asm/agp.h>
53#endif
Pauli Nieminen1403b1a2010-04-01 12:44:57 +000054
55#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *))
56#define SMALL_ALLOCATION 16
57#define FREE_ALL_PAGES (~0U)
58/* times are in msecs */
59#define PAGE_FREE_INTERVAL 1000
60
61/**
62 * struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages.
63 *
64 * @lock: Protects the shared pool from concurrnet access. Must be used with
65 * irqsave/irqrestore variants because pool allocator maybe called from
66 * delayed work.
67 * @fill_lock: Prevent concurrent calls to fill.
68 * @list: Pool of free uc/wc pages for fast reuse.
69 * @gfp_flags: Flags to pass for alloc_page.
70 * @npages: Number of pages in pool.
71 */
72struct ttm_page_pool {
73 spinlock_t lock;
74 bool fill_lock;
75 struct list_head list;
Daniel J Blueman0e57a3c2010-09-22 17:45:45 +010076 gfp_t gfp_flags;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +000077 unsigned npages;
Pauli Nieminen07458662010-04-01 12:44:58 +000078 char *name;
79 unsigned long nfrees;
80 unsigned long nrefills;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +000081};
82
Pauli Nieminenc96af792010-04-01 12:45:03 +000083/**
84 * Limits for the pool. They are handled without locks because only place where
85 * they may change is in sysfs store. They won't have immediate effect anyway
Thomas Hellstrom4abe4382010-05-26 16:21:04 +020086 * so forcing serialization to access them is pointless.
Pauli Nieminenc96af792010-04-01 12:45:03 +000087 */
88
Pauli Nieminen1403b1a2010-04-01 12:44:57 +000089struct ttm_pool_opts {
90 unsigned alloc_size;
91 unsigned max_size;
92 unsigned small;
93};
94
95#define NUM_POOLS 4
96
97/**
98 * struct ttm_pool_manager - Holds memory pools for fst allocation
99 *
100 * Manager is read only object for pool code so it doesn't need locking.
101 *
102 * @free_interval: minimum number of jiffies between freeing pages from pool.
103 * @page_alloc_inited: reference counting for pool allocation.
104 * @work: Work that is used to shrink the pool. Work is only run when there is
105 * some pages to free.
106 * @small_allocation: Limit in number of pages what is small allocation.
107 *
108 * @pools: All pool objects in use.
109 **/
110struct ttm_pool_manager {
Pauli Nieminenc96af792010-04-01 12:45:03 +0000111 struct kobject kobj;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000112 struct shrinker mm_shrink;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000113 struct ttm_pool_opts options;
114
115 union {
116 struct ttm_page_pool pools[NUM_POOLS];
117 struct {
118 struct ttm_page_pool wc_pool;
119 struct ttm_page_pool uc_pool;
120 struct ttm_page_pool wc_pool_dma32;
121 struct ttm_page_pool uc_pool_dma32;
122 } ;
123 };
124};
125
Pauli Nieminenc96af792010-04-01 12:45:03 +0000126static struct attribute ttm_page_pool_max = {
127 .name = "pool_max_size",
128 .mode = S_IRUGO | S_IWUSR
129};
130static struct attribute ttm_page_pool_small = {
131 .name = "pool_small_allocation",
132 .mode = S_IRUGO | S_IWUSR
133};
134static struct attribute ttm_page_pool_alloc_size = {
135 .name = "pool_allocation_size",
136 .mode = S_IRUGO | S_IWUSR
137};
138
139static struct attribute *ttm_pool_attrs[] = {
140 &ttm_page_pool_max,
141 &ttm_page_pool_small,
142 &ttm_page_pool_alloc_size,
143 NULL
144};
145
146static void ttm_pool_kobj_release(struct kobject *kobj)
147{
148 struct ttm_pool_manager *m =
149 container_of(kobj, struct ttm_pool_manager, kobj);
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200150 kfree(m);
Pauli Nieminenc96af792010-04-01 12:45:03 +0000151}
152
153static ssize_t ttm_pool_store(struct kobject *kobj,
154 struct attribute *attr, const char *buffer, size_t size)
155{
156 struct ttm_pool_manager *m =
157 container_of(kobj, struct ttm_pool_manager, kobj);
158 int chars;
159 unsigned val;
160 chars = sscanf(buffer, "%u", &val);
161 if (chars == 0)
162 return size;
163
164 /* Convert kb to number of pages */
165 val = val / (PAGE_SIZE >> 10);
166
167 if (attr == &ttm_page_pool_max)
168 m->options.max_size = val;
169 else if (attr == &ttm_page_pool_small)
170 m->options.small = val;
171 else if (attr == &ttm_page_pool_alloc_size) {
172 if (val > NUM_PAGES_TO_ALLOC*8) {
Joe Perches25d04792012-03-16 21:43:50 -0700173 pr_err("Setting allocation size to %lu is not allowed. Recommended size is %lu\n",
Thomas Hellstrom4abe4382010-05-26 16:21:04 +0200174 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7),
175 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
Pauli Nieminenc96af792010-04-01 12:45:03 +0000176 return size;
177 } else if (val > NUM_PAGES_TO_ALLOC) {
Joe Perches25d04792012-03-16 21:43:50 -0700178 pr_warn("Setting allocation size to larger than %lu is not recommended\n",
179 NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
Pauli Nieminenc96af792010-04-01 12:45:03 +0000180 }
181 m->options.alloc_size = val;
182 }
183
184 return size;
185}
186
187static ssize_t ttm_pool_show(struct kobject *kobj,
188 struct attribute *attr, char *buffer)
189{
190 struct ttm_pool_manager *m =
191 container_of(kobj, struct ttm_pool_manager, kobj);
192 unsigned val = 0;
193
194 if (attr == &ttm_page_pool_max)
195 val = m->options.max_size;
196 else if (attr == &ttm_page_pool_small)
197 val = m->options.small;
198 else if (attr == &ttm_page_pool_alloc_size)
199 val = m->options.alloc_size;
200
201 val = val * (PAGE_SIZE >> 10);
202
203 return snprintf(buffer, PAGE_SIZE, "%u\n", val);
204}
205
206static const struct sysfs_ops ttm_pool_sysfs_ops = {
207 .show = &ttm_pool_show,
208 .store = &ttm_pool_store,
209};
210
211static struct kobj_type ttm_pool_kobj_type = {
212 .release = &ttm_pool_kobj_release,
213 .sysfs_ops = &ttm_pool_sysfs_ops,
214 .default_attrs = ttm_pool_attrs,
215};
216
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200217static struct ttm_pool_manager *_manager;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000218
Pauli Nieminen975efdb2010-04-01 12:45:02 +0000219#ifndef CONFIG_X86
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000220static int set_pages_array_wb(struct page **pages, int addrinarray)
221{
222#ifdef TTM_HAS_AGP
223 int i;
224
225 for (i = 0; i < addrinarray; i++)
226 unmap_page_from_agp(pages[i]);
227#endif
228 return 0;
229}
230
231static int set_pages_array_wc(struct page **pages, int addrinarray)
232{
233#ifdef TTM_HAS_AGP
234 int i;
235
236 for (i = 0; i < addrinarray; i++)
237 map_page_into_agp(pages[i]);
238#endif
239 return 0;
240}
241
242static int set_pages_array_uc(struct page **pages, int addrinarray)
243{
244#ifdef TTM_HAS_AGP
245 int i;
246
247 for (i = 0; i < addrinarray; i++)
248 map_page_into_agp(pages[i]);
249#endif
250 return 0;
251}
252#endif
253
254/**
255 * Select the right pool or requested caching state and ttm flags. */
256static struct ttm_page_pool *ttm_get_pool(int flags,
257 enum ttm_caching_state cstate)
258{
259 int pool_index;
260
261 if (cstate == tt_cached)
262 return NULL;
263
264 if (cstate == tt_wc)
265 pool_index = 0x0;
266 else
267 pool_index = 0x1;
268
269 if (flags & TTM_PAGE_FLAG_DMA32)
270 pool_index |= 0x2;
271
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200272 return &_manager->pools[pool_index];
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000273}
274
275/* set memory back to wb and free the pages. */
276static void ttm_pages_put(struct page *pages[], unsigned npages)
277{
278 unsigned i;
279 if (set_pages_array_wb(pages, npages))
Joe Perches25d04792012-03-16 21:43:50 -0700280 pr_err("Failed to set %d pages to wb!\n", npages);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000281 for (i = 0; i < npages; ++i)
282 __free_page(pages[i]);
283}
284
285static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
286 unsigned freed_pages)
287{
288 pool->npages -= freed_pages;
Pauli Nieminen07458662010-04-01 12:44:58 +0000289 pool->nfrees += freed_pages;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000290}
291
292/**
293 * Free pages from pool.
294 *
295 * To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC
296 * number of pages in one go.
297 *
298 * @pool: to free the pages from
299 * @free_all: If set to true will free all pages in pool
Tetsuo Handaa91576d2014-08-03 20:02:31 +0900300 * @gfp: GFP flags.
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000301 **/
Tetsuo Handaa91576d2014-08-03 20:02:31 +0900302static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free,
303 gfp_t gfp)
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000304{
305 unsigned long irq_flags;
306 struct page *p;
307 struct page **pages_to_free;
308 unsigned freed_pages = 0,
309 npages_to_free = nr_free;
310
311 if (NUM_PAGES_TO_ALLOC < nr_free)
312 npages_to_free = NUM_PAGES_TO_ALLOC;
313
Tetsuo Handaa91576d2014-08-03 20:02:31 +0900314 pages_to_free = kmalloc(npages_to_free * sizeof(struct page *), gfp);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000315 if (!pages_to_free) {
Joe Perches25d04792012-03-16 21:43:50 -0700316 pr_err("Failed to allocate memory for pool free operation\n");
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000317 return 0;
318 }
319
320restart:
321 spin_lock_irqsave(&pool->lock, irq_flags);
322
323 list_for_each_entry_reverse(p, &pool->list, lru) {
324 if (freed_pages >= npages_to_free)
325 break;
326
327 pages_to_free[freed_pages++] = p;
328 /* We can only remove NUM_PAGES_TO_ALLOC at a time. */
329 if (freed_pages >= NUM_PAGES_TO_ALLOC) {
330 /* remove range of pages from the pool */
331 __list_del(p->lru.prev, &pool->list);
332
333 ttm_pool_update_free_locked(pool, freed_pages);
334 /**
335 * Because changing page caching is costly
336 * we unlock the pool to prevent stalling.
337 */
338 spin_unlock_irqrestore(&pool->lock, irq_flags);
339
340 ttm_pages_put(pages_to_free, freed_pages);
341 if (likely(nr_free != FREE_ALL_PAGES))
342 nr_free -= freed_pages;
343
344 if (NUM_PAGES_TO_ALLOC >= nr_free)
345 npages_to_free = nr_free;
346 else
347 npages_to_free = NUM_PAGES_TO_ALLOC;
348
349 freed_pages = 0;
350
351 /* free all so restart the processing */
352 if (nr_free)
353 goto restart;
354
Konrad Rzeszutek Wilk0d74f862011-06-08 17:06:15 +0000355 /* Not allowed to fall through or break because
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000356 * following context is inside spinlock while we are
357 * outside here.
358 */
359 goto out;
360
361 }
362 }
363
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000364 /* remove range of pages from the pool */
365 if (freed_pages) {
366 __list_del(&p->lru, &pool->list);
367
368 ttm_pool_update_free_locked(pool, freed_pages);
369 nr_free -= freed_pages;
370 }
371
372 spin_unlock_irqrestore(&pool->lock, irq_flags);
373
374 if (freed_pages)
375 ttm_pages_put(pages_to_free, freed_pages);
376out:
377 kfree(pages_to_free);
378 return nr_free;
379}
380
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000381/**
Thomas Hellstrom4abe4382010-05-26 16:21:04 +0200382 * Callback for mm to request pool to reduce number of page held.
Dave Chinner7dc19d52013-08-28 10:18:11 +1000383 *
384 * XXX: (dchinner) Deadlock warning!
385 *
Tetsuo Handaa91576d2014-08-03 20:02:31 +0900386 * We need to pass sc->gfp_mask to ttm_page_pool_free().
Dave Chinner7dc19d52013-08-28 10:18:11 +1000387 *
388 * This code is crying out for a shrinker per pool....
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000389 */
Dave Chinner7dc19d52013-08-28 10:18:11 +1000390static unsigned long
391ttm_pool_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000392{
Tetsuo Handa71336e02014-08-03 20:02:03 +0900393 static DEFINE_MUTEX(lock);
394 static unsigned start_pool;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000395 unsigned i;
Tetsuo Handa71336e02014-08-03 20:02:03 +0900396 unsigned pool_offset;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000397 struct ttm_page_pool *pool;
Ying Han1495f232011-05-24 17:12:27 -0700398 int shrink_pages = sc->nr_to_scan;
Dave Chinner7dc19d52013-08-28 10:18:11 +1000399 unsigned long freed = 0;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000400
Tetsuo Handa71336e02014-08-03 20:02:03 +0900401 if (!mutex_trylock(&lock))
402 return SHRINK_STOP;
403 pool_offset = ++start_pool % NUM_POOLS;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000404 /* select start pool in round robin fashion */
405 for (i = 0; i < NUM_POOLS; ++i) {
406 unsigned nr_free = shrink_pages;
407 if (shrink_pages == 0)
408 break;
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200409 pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
Tetsuo Handaa91576d2014-08-03 20:02:31 +0900410 shrink_pages = ttm_page_pool_free(pool, nr_free,
411 sc->gfp_mask);
Dave Chinner7dc19d52013-08-28 10:18:11 +1000412 freed += nr_free - shrink_pages;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000413 }
Tetsuo Handa71336e02014-08-03 20:02:03 +0900414 mutex_unlock(&lock);
Dave Chinner7dc19d52013-08-28 10:18:11 +1000415 return freed;
416}
417
418
419static unsigned long
420ttm_pool_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
421{
422 unsigned i;
423 unsigned long count = 0;
424
425 for (i = 0; i < NUM_POOLS; ++i)
426 count += _manager->pools[i].npages;
427
428 return count;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000429}
430
431static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
432{
Dave Chinner7dc19d52013-08-28 10:18:11 +1000433 manager->mm_shrink.count_objects = ttm_pool_shrink_count;
434 manager->mm_shrink.scan_objects = ttm_pool_shrink_scan;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000435 manager->mm_shrink.seeks = 1;
436 register_shrinker(&manager->mm_shrink);
437}
438
439static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
440{
441 unregister_shrinker(&manager->mm_shrink);
442}
443
444static int ttm_set_pages_caching(struct page **pages,
445 enum ttm_caching_state cstate, unsigned cpages)
446{
447 int r = 0;
448 /* Set page caching */
449 switch (cstate) {
450 case tt_uncached:
451 r = set_pages_array_uc(pages, cpages);
452 if (r)
Joe Perches25d04792012-03-16 21:43:50 -0700453 pr_err("Failed to set %d pages to uc!\n", cpages);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000454 break;
455 case tt_wc:
456 r = set_pages_array_wc(pages, cpages);
457 if (r)
Joe Perches25d04792012-03-16 21:43:50 -0700458 pr_err("Failed to set %d pages to wc!\n", cpages);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000459 break;
460 default:
461 break;
462 }
463 return r;
464}
465
466/**
467 * Free pages the pages that failed to change the caching state. If there is
468 * any pages that have changed their caching state already put them to the
469 * pool.
470 */
471static void ttm_handle_caching_state_failure(struct list_head *pages,
472 int ttm_flags, enum ttm_caching_state cstate,
473 struct page **failed_pages, unsigned cpages)
474{
475 unsigned i;
Thomas Hellstrom4abe4382010-05-26 16:21:04 +0200476 /* Failed pages have to be freed */
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000477 for (i = 0; i < cpages; ++i) {
478 list_del(&failed_pages[i]->lru);
479 __free_page(failed_pages[i]);
480 }
481}
482
483/**
484 * Allocate new pages with correct caching.
485 *
486 * This function is reentrant if caller updates count depending on number of
487 * pages returned in pages array.
488 */
Daniel J Blueman0e57a3c2010-09-22 17:45:45 +0100489static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000490 int ttm_flags, enum ttm_caching_state cstate, unsigned count)
491{
492 struct page **caching_array;
493 struct page *p;
494 int r = 0;
495 unsigned i, cpages;
496 unsigned max_cpages = min(count,
497 (unsigned)(PAGE_SIZE/sizeof(struct page *)));
498
499 /* allocate array for page caching change */
500 caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL);
501
502 if (!caching_array) {
Joe Perches25d04792012-03-16 21:43:50 -0700503 pr_err("Unable to allocate table for new pages\n");
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000504 return -ENOMEM;
505 }
506
507 for (i = 0, cpages = 0; i < count; ++i) {
508 p = alloc_page(gfp_flags);
509
510 if (!p) {
Joe Perches25d04792012-03-16 21:43:50 -0700511 pr_err("Unable to get page %u\n", i);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000512
513 /* store already allocated pages in the pool after
514 * setting the caching state */
515 if (cpages) {
Thomas Hellstrom4abe4382010-05-26 16:21:04 +0200516 r = ttm_set_pages_caching(caching_array,
517 cstate, cpages);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000518 if (r)
519 ttm_handle_caching_state_failure(pages,
520 ttm_flags, cstate,
521 caching_array, cpages);
522 }
523 r = -ENOMEM;
524 goto out;
525 }
526
527#ifdef CONFIG_HIGHMEM
528 /* gfp flags of highmem page should never be dma32 so we
529 * we should be fine in such case
530 */
531 if (!PageHighMem(p))
532#endif
533 {
534 caching_array[cpages++] = p;
535 if (cpages == max_cpages) {
536
537 r = ttm_set_pages_caching(caching_array,
538 cstate, cpages);
539 if (r) {
540 ttm_handle_caching_state_failure(pages,
541 ttm_flags, cstate,
542 caching_array, cpages);
543 goto out;
544 }
545 cpages = 0;
546 }
547 }
548
549 list_add(&p->lru, pages);
550 }
551
552 if (cpages) {
553 r = ttm_set_pages_caching(caching_array, cstate, cpages);
554 if (r)
555 ttm_handle_caching_state_failure(pages,
556 ttm_flags, cstate,
557 caching_array, cpages);
558 }
559out:
560 kfree(caching_array);
561
562 return r;
563}
564
565/**
Konrad Rzeszutek Wilk0d74f862011-06-08 17:06:15 +0000566 * Fill the given pool if there aren't enough pages and the requested number of
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000567 * pages is small.
568 */
569static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
570 int ttm_flags, enum ttm_caching_state cstate, unsigned count,
571 unsigned long *irq_flags)
572{
573 struct page *p;
574 int r;
575 unsigned cpages = 0;
576 /**
577 * Only allow one pool fill operation at a time.
578 * If pool doesn't have enough pages for the allocation new pages are
579 * allocated from outside of pool.
580 */
581 if (pool->fill_lock)
582 return;
583
584 pool->fill_lock = true;
585
Konrad Rzeszutek Wilk0d74f862011-06-08 17:06:15 +0000586 /* If allocation request is small and there are not enough
587 * pages in a pool we fill the pool up first. */
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200588 if (count < _manager->options.small
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000589 && count > pool->npages) {
590 struct list_head new_pages;
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200591 unsigned alloc_size = _manager->options.alloc_size;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000592
593 /**
594 * Can't change page caching if in irqsave context. We have to
595 * drop the pool->lock.
596 */
597 spin_unlock_irqrestore(&pool->lock, *irq_flags);
598
599 INIT_LIST_HEAD(&new_pages);
600 r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
601 cstate, alloc_size);
602 spin_lock_irqsave(&pool->lock, *irq_flags);
603
604 if (!r) {
605 list_splice(&new_pages, &pool->list);
Pauli Nieminen07458662010-04-01 12:44:58 +0000606 ++pool->nrefills;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000607 pool->npages += alloc_size;
608 } else {
Joe Perches25d04792012-03-16 21:43:50 -0700609 pr_err("Failed to fill pool (%p)\n", pool);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000610 /* If we have any pages left put them to the pool. */
611 list_for_each_entry(p, &pool->list, lru) {
612 ++cpages;
613 }
614 list_splice(&new_pages, &pool->list);
615 pool->npages += cpages;
616 }
617
618 }
619 pool->fill_lock = false;
620}
621
622/**
Konrad Rzeszutek Wilk0d74f862011-06-08 17:06:15 +0000623 * Cut 'count' number of pages from the pool and put them on the return list.
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000624 *
Konrad Rzeszutek Wilk0d74f862011-06-08 17:06:15 +0000625 * @return count of pages still required to fulfill the request.
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000626 */
627static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
Jerome Glisse822c4d92011-11-10 18:24:09 -0500628 struct list_head *pages,
629 int ttm_flags,
630 enum ttm_caching_state cstate,
631 unsigned count)
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000632{
633 unsigned long irq_flags;
634 struct list_head *p;
635 unsigned i;
636
637 spin_lock_irqsave(&pool->lock, irq_flags);
638 ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);
639
640 if (count >= pool->npages) {
641 /* take all pages from the pool */
642 list_splice_init(&pool->list, pages);
643 count -= pool->npages;
644 pool->npages = 0;
645 goto out;
646 }
647 /* find the last pages to include for requested number of pages. Split
Konrad Rzeszutek Wilk0d74f862011-06-08 17:06:15 +0000648 * pool to begin and halve it to reduce search space. */
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000649 if (count <= pool->npages/2) {
650 i = 0;
651 list_for_each(p, &pool->list) {
652 if (++i == count)
653 break;
654 }
655 } else {
656 i = pool->npages + 1;
657 list_for_each_prev(p, &pool->list) {
658 if (--i == count)
659 break;
660 }
661 }
Konrad Rzeszutek Wilk0d74f862011-06-08 17:06:15 +0000662 /* Cut 'count' number of pages from the pool */
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000663 list_cut_position(pages, &pool->list, p);
664 pool->npages -= count;
665 count = 0;
666out:
667 spin_unlock_irqrestore(&pool->lock, irq_flags);
668 return count;
669}
670
Jerome Glisse8e7e7052011-11-09 17:15:26 -0500671/* Put all pages in pages list to correct pool to wait for reuse */
672static void ttm_put_pages(struct page **pages, unsigned npages, int flags,
673 enum ttm_caching_state cstate)
674{
675 unsigned long irq_flags;
676 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
677 unsigned i;
678
679 if (pool == NULL) {
680 /* No pool for this memory type so free the pages */
681 for (i = 0; i < npages; i++) {
682 if (pages[i]) {
683 if (page_count(pages[i]) != 1)
Joe Perches25d04792012-03-16 21:43:50 -0700684 pr_err("Erroneous page count. Leaking pages.\n");
Jerome Glisse8e7e7052011-11-09 17:15:26 -0500685 __free_page(pages[i]);
686 pages[i] = NULL;
687 }
688 }
689 return;
690 }
691
692 spin_lock_irqsave(&pool->lock, irq_flags);
693 for (i = 0; i < npages; i++) {
694 if (pages[i]) {
695 if (page_count(pages[i]) != 1)
Joe Perches25d04792012-03-16 21:43:50 -0700696 pr_err("Erroneous page count. Leaking pages.\n");
Jerome Glisse8e7e7052011-11-09 17:15:26 -0500697 list_add_tail(&pages[i]->lru, &pool->list);
698 pages[i] = NULL;
699 pool->npages++;
700 }
701 }
702 /* Check that we don't go over the pool limit */
703 npages = 0;
704 if (pool->npages > _manager->options.max_size) {
705 npages = pool->npages - _manager->options.max_size;
706 /* free at least NUM_PAGES_TO_ALLOC number of pages
707 * to reduce calls to set_memory_wb */
708 if (npages < NUM_PAGES_TO_ALLOC)
709 npages = NUM_PAGES_TO_ALLOC;
710 }
711 spin_unlock_irqrestore(&pool->lock, irq_flags);
712 if (npages)
Tetsuo Handaa91576d2014-08-03 20:02:31 +0900713 ttm_page_pool_free(pool, npages, GFP_KERNEL);
Jerome Glisse8e7e7052011-11-09 17:15:26 -0500714}
715
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000716/*
717 * On success pages list will hold count number of correctly
718 * cached pages.
719 */
Jerome Glisse8e7e7052011-11-09 17:15:26 -0500720static int ttm_get_pages(struct page **pages, unsigned npages, int flags,
721 enum ttm_caching_state cstate)
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000722{
723 struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
Jerome Glisse822c4d92011-11-10 18:24:09 -0500724 struct list_head plist;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000725 struct page *p = NULL;
Daniel J Blueman0e57a3c2010-09-22 17:45:45 +0100726 gfp_t gfp_flags = GFP_USER;
Jerome Glisse822c4d92011-11-10 18:24:09 -0500727 unsigned count;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000728 int r;
729
730 /* set zero flag for page allocation if required */
731 if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
732 gfp_flags |= __GFP_ZERO;
733
734 /* No pool for cached pages */
735 if (pool == NULL) {
736 if (flags & TTM_PAGE_FLAG_DMA32)
737 gfp_flags |= GFP_DMA32;
738 else
Thomas Hellstrome8613c02010-05-26 16:21:03 +0200739 gfp_flags |= GFP_HIGHUSER;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000740
Jerome Glisse822c4d92011-11-10 18:24:09 -0500741 for (r = 0; r < npages; ++r) {
Dave Airlied87dfdb2011-04-13 09:15:09 +1000742 p = alloc_page(gfp_flags);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000743 if (!p) {
744
Joe Perches25d04792012-03-16 21:43:50 -0700745 pr_err("Unable to allocate page\n");
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000746 return -ENOMEM;
747 }
Dave Airlied87dfdb2011-04-13 09:15:09 +1000748
Jerome Glisse822c4d92011-11-10 18:24:09 -0500749 pages[r] = p;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000750 }
751 return 0;
752 }
753
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000754 /* combine zero flag to pool flags */
755 gfp_flags |= pool->gfp_flags;
756
757 /* First we take pages from the pool */
Jerome Glisse822c4d92011-11-10 18:24:09 -0500758 INIT_LIST_HEAD(&plist);
759 npages = ttm_page_pool_get_pages(pool, &plist, flags, cstate, npages);
760 count = 0;
761 list_for_each_entry(p, &plist, lru) {
762 pages[count++] = p;
763 }
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000764
765 /* clear the pages coming from the pool if requested */
766 if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
Jerome Glisse822c4d92011-11-10 18:24:09 -0500767 list_for_each_entry(p, &plist, lru) {
Zhao Yakuiac207ed2012-11-13 18:31:55 +0000768 if (PageHighMem(p))
769 clear_highpage(p);
770 else
771 clear_page(page_address(p));
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000772 }
773 }
774
775 /* If pool didn't have enough pages allocate new one. */
Jerome Glisse822c4d92011-11-10 18:24:09 -0500776 if (npages > 0) {
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000777 /* ttm_alloc_new_pages doesn't reference pool so we can run
778 * multiple requests in parallel.
779 **/
Jerome Glisse822c4d92011-11-10 18:24:09 -0500780 INIT_LIST_HEAD(&plist);
781 r = ttm_alloc_new_pages(&plist, gfp_flags, flags, cstate, npages);
782 list_for_each_entry(p, &plist, lru) {
783 pages[count++] = p;
784 }
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000785 if (r) {
786 /* If there is any pages in the list put them back to
787 * the pool. */
Joe Perches25d04792012-03-16 21:43:50 -0700788 pr_err("Failed to allocate extra pages for large request\n");
Jerome Glisse8e7e7052011-11-09 17:15:26 -0500789 ttm_put_pages(pages, count, flags, cstate);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000790 return r;
791 }
792 }
793
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000794 return 0;
795}
796
Dave Airlie3b9c2142014-07-22 12:55:39 +1000797static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, gfp_t flags,
Pauli Nieminen07458662010-04-01 12:44:58 +0000798 char *name)
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000799{
800 spin_lock_init(&pool->lock);
801 pool->fill_lock = false;
802 INIT_LIST_HEAD(&pool->list);
Pauli Nieminen07458662010-04-01 12:44:58 +0000803 pool->npages = pool->nfrees = 0;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000804 pool->gfp_flags = flags;
Pauli Nieminen07458662010-04-01 12:44:58 +0000805 pool->name = name;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000806}
807
Pauli Nieminenc96af792010-04-01 12:45:03 +0000808int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000809{
Pauli Nieminenc96af792010-04-01 12:45:03 +0000810 int ret;
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200811
812 WARN_ON(_manager);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000813
Joe Perches25d04792012-03-16 21:43:50 -0700814 pr_info("Initializing pool allocator\n");
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000815
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200816 _manager = kzalloc(sizeof(*_manager), GFP_KERNEL);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000817
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200818 ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc");
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000819
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200820 ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc");
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000821
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200822 ttm_page_pool_init_locked(&_manager->wc_pool_dma32,
823 GFP_USER | GFP_DMA32, "wc dma");
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000824
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200825 ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
826 GFP_USER | GFP_DMA32, "uc dma");
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000827
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200828 _manager->options.max_size = max_pages;
829 _manager->options.small = SMALL_ALLOCATION;
830 _manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
831
832 ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type,
833 &glob->kobj, "pool");
Pauli Nieminenc96af792010-04-01 12:45:03 +0000834 if (unlikely(ret != 0)) {
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200835 kobject_put(&_manager->kobj);
836 _manager = NULL;
Pauli Nieminenc96af792010-04-01 12:45:03 +0000837 return ret;
838 }
839
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200840 ttm_pool_mm_shrink_init(_manager);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000841
842 return 0;
843}
844
Daniel J Blueman0e57a3c2010-09-22 17:45:45 +0100845void ttm_page_alloc_fini(void)
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000846{
847 int i;
848
Joe Perches25d04792012-03-16 21:43:50 -0700849 pr_info("Finalizing pool allocator\n");
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200850 ttm_pool_mm_shrink_fini(_manager);
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000851
852 for (i = 0; i < NUM_POOLS; ++i)
Tetsuo Handaa91576d2014-08-03 20:02:31 +0900853 ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES,
854 GFP_KERNEL);
Pauli Nieminenc96af792010-04-01 12:45:03 +0000855
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200856 kobject_put(&_manager->kobj);
857 _manager = NULL;
Pauli Nieminen1403b1a2010-04-01 12:44:57 +0000858}
Pauli Nieminen07458662010-04-01 12:44:58 +0000859
Jerome Glisseb1e5f172011-11-02 23:59:28 -0400860int ttm_pool_populate(struct ttm_tt *ttm)
861{
862 struct ttm_mem_global *mem_glob = ttm->glob->mem_glob;
863 unsigned i;
864 int ret;
865
866 if (ttm->state != tt_unpopulated)
867 return 0;
868
869 for (i = 0; i < ttm->num_pages; ++i) {
Jerome Glisse8e7e7052011-11-09 17:15:26 -0500870 ret = ttm_get_pages(&ttm->pages[i], 1,
871 ttm->page_flags,
872 ttm->caching_state);
Jerome Glisseb1e5f172011-11-02 23:59:28 -0400873 if (ret != 0) {
874 ttm_pool_unpopulate(ttm);
875 return -ENOMEM;
876 }
877
878 ret = ttm_mem_global_alloc_page(mem_glob, ttm->pages[i],
879 false, false);
880 if (unlikely(ret != 0)) {
881 ttm_pool_unpopulate(ttm);
882 return -ENOMEM;
883 }
884 }
885
886 if (unlikely(ttm->page_flags & TTM_PAGE_FLAG_SWAPPED)) {
887 ret = ttm_tt_swapin(ttm);
888 if (unlikely(ret != 0)) {
889 ttm_pool_unpopulate(ttm);
890 return ret;
891 }
892 }
893
894 ttm->state = tt_unbound;
895 return 0;
896}
897EXPORT_SYMBOL(ttm_pool_populate);
898
899void ttm_pool_unpopulate(struct ttm_tt *ttm)
900{
901 unsigned i;
902
903 for (i = 0; i < ttm->num_pages; ++i) {
904 if (ttm->pages[i]) {
905 ttm_mem_global_free_page(ttm->glob->mem_glob,
906 ttm->pages[i]);
907 ttm_put_pages(&ttm->pages[i], 1,
908 ttm->page_flags,
Jerome Glisse8e7e7052011-11-09 17:15:26 -0500909 ttm->caching_state);
Jerome Glisseb1e5f172011-11-02 23:59:28 -0400910 }
911 }
912 ttm->state = tt_unpopulated;
913}
914EXPORT_SYMBOL(ttm_pool_unpopulate);
915
Pauli Nieminen07458662010-04-01 12:44:58 +0000916int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
917{
918 struct ttm_page_pool *p;
919 unsigned i;
920 char *h[] = {"pool", "refills", "pages freed", "size"};
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200921 if (!_manager) {
Pauli Nieminen07458662010-04-01 12:44:58 +0000922 seq_printf(m, "No pool allocator running.\n");
923 return 0;
924 }
925 seq_printf(m, "%6s %12s %13s %8s\n",
926 h[0], h[1], h[2], h[3]);
927 for (i = 0; i < NUM_POOLS; ++i) {
Francisco Jerez5870a4d2010-07-04 04:03:07 +0200928 p = &_manager->pools[i];
Pauli Nieminen07458662010-04-01 12:44:58 +0000929
930 seq_printf(m, "%6s %12ld %13ld %8d\n",
931 p->name, p->nrefills,
932 p->nfrees, p->npages);
933 }
934 return 0;
935}
936EXPORT_SYMBOL(ttm_page_alloc_debugfs);