Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | /* SPDX-License-Identifier: GPL-2.0 */ |
Christoph Lameter | 6225e93 | 2007-05-06 14:49:50 -0700 | [diff] [blame] | 2 | #ifndef LINUX_QUICKLIST_H |
| 3 | #define LINUX_QUICKLIST_H |
| 4 | /* |
| 5 | * Fast allocations and disposal of pages. Pages must be in the condition |
| 6 | * as needed after allocation when they are freed. Per cpu lists of pages |
| 7 | * are kept that only contain node local pages. |
| 8 | * |
Christoph Lameter | 93e205a | 2016-03-17 14:21:15 -0700 | [diff] [blame] | 9 | * (C) 2007, SGI. Christoph Lameter <cl@linux.com> |
Christoph Lameter | 6225e93 | 2007-05-06 14:49:50 -0700 | [diff] [blame] | 10 | */ |
| 11 | #include <linux/kernel.h> |
| 12 | #include <linux/gfp.h> |
| 13 | #include <linux/percpu.h> |
| 14 | |
| 15 | #ifdef CONFIG_QUICKLIST |
| 16 | |
| 17 | struct quicklist { |
| 18 | void *page; |
| 19 | int nr_pages; |
| 20 | }; |
| 21 | |
| 22 | DECLARE_PER_CPU(struct quicklist, quicklist)[CONFIG_NR_QUICK]; |
| 23 | |
| 24 | /* |
| 25 | * The two key functions quicklist_alloc and quicklist_free are inline so |
| 26 | * that they may be custom compiled for the platform. |
| 27 | * Specifying a NULL ctor can remove constructor support. Specifying |
| 28 | * a constant quicklist allows the determination of the exact address |
| 29 | * in the per cpu area. |
| 30 | * |
| 31 | * The fast patch in quicklist_alloc touched only a per cpu cacheline and |
| 32 | * the first cacheline of the page itself. There is minmal overhead involved. |
| 33 | */ |
| 34 | static inline void *quicklist_alloc(int nr, gfp_t flags, void (*ctor)(void *)) |
| 35 | { |
| 36 | struct quicklist *q; |
| 37 | void **p = NULL; |
| 38 | |
| 39 | q =&get_cpu_var(quicklist)[nr]; |
| 40 | p = q->page; |
| 41 | if (likely(p)) { |
| 42 | q->page = p[0]; |
| 43 | p[0] = NULL; |
| 44 | q->nr_pages--; |
| 45 | } |
| 46 | put_cpu_var(quicklist); |
| 47 | if (likely(p)) |
| 48 | return p; |
| 49 | |
| 50 | p = (void *)__get_free_page(flags | __GFP_ZERO); |
| 51 | if (ctor && p) |
| 52 | ctor(p); |
| 53 | return p; |
| 54 | } |
| 55 | |
| 56 | static inline void __quicklist_free(int nr, void (*dtor)(void *), void *p, |
| 57 | struct page *page) |
| 58 | { |
| 59 | struct quicklist *q; |
Christoph Lameter | 6225e93 | 2007-05-06 14:49:50 -0700 | [diff] [blame] | 60 | |
| 61 | q = &get_cpu_var(quicklist)[nr]; |
| 62 | *(void **)p = q->page; |
| 63 | q->page = p; |
| 64 | q->nr_pages++; |
| 65 | put_cpu_var(quicklist); |
| 66 | } |
| 67 | |
| 68 | static inline void quicklist_free(int nr, void (*dtor)(void *), void *pp) |
| 69 | { |
| 70 | __quicklist_free(nr, dtor, pp, virt_to_page(pp)); |
| 71 | } |
| 72 | |
| 73 | static inline void quicklist_free_page(int nr, void (*dtor)(void *), |
| 74 | struct page *page) |
| 75 | { |
| 76 | __quicklist_free(nr, dtor, page_address(page), page); |
| 77 | } |
| 78 | |
| 79 | void quicklist_trim(int nr, void (*dtor)(void *), |
| 80 | unsigned long min_pages, unsigned long max_free); |
| 81 | |
| 82 | unsigned long quicklist_total_size(void); |
| 83 | |
KOSAKI Motohiro | 4b85615 | 2008-09-02 14:35:53 -0700 | [diff] [blame] | 84 | #else |
| 85 | |
| 86 | static inline unsigned long quicklist_total_size(void) |
| 87 | { |
| 88 | return 0; |
| 89 | } |
| 90 | |
Christoph Lameter | 6225e93 | 2007-05-06 14:49:50 -0700 | [diff] [blame] | 91 | #endif |
| 92 | |
| 93 | #endif /* LINUX_QUICKLIST_H */ |
| 94 | |