blob: b1ccb58ad397403214a220e4a0ac7901a6b6ae1e [file] [log] [blame]
Tim Chen67afa382017-02-22 15:45:39 -08001/*
2 * Manage cache of swap slots to be used for and returned from
3 * swap.
4 *
5 * Copyright(c) 2016 Intel Corporation.
6 *
7 * Author: Tim Chen <tim.c.chen@linux.intel.com>
8 *
9 * We allocate the swap slots from the global pool and put
10 * it into local per cpu caches. This has the advantage
11 * of no needing to acquire the swap_info lock every time
12 * we need a new slot.
13 *
14 * There is also opportunity to simply return the slot
15 * to local caches without needing to acquire swap_info
16 * lock. We do not reuse the returned slots directly but
17 * move them back to the global pool in a batch. This
18 * allows the slots to coaellesce and reduce fragmentation.
19 *
20 * The swap entry allocated is marked with SWAP_HAS_CACHE
21 * flag in map_count that prevents it from being allocated
22 * again from the global pool.
23 *
24 * The swap slots cache is protected by a mutex instead of
25 * a spin lock as when we search for slots with scan_swap_map,
26 * we can possibly sleep.
27 */
28
29#include <linux/swap_slots.h>
30#include <linux/cpu.h>
31#include <linux/cpumask.h>
32#include <linux/vmalloc.h>
33#include <linux/mutex.h>
34
35#ifdef CONFIG_SWAP
36
37static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
38static bool swap_slot_cache_active;
Huang Yingba81f832017-02-22 15:45:46 -080039bool swap_slot_cache_enabled;
Tim Chen67afa382017-02-22 15:45:39 -080040static bool swap_slot_cache_initialized;
41DEFINE_MUTEX(swap_slots_cache_mutex);
42/* Serialize swap slots cache enable/disable operations */
43DEFINE_MUTEX(swap_slots_cache_enable_mutex);
44
45static void __drain_swap_slots_cache(unsigned int type);
46static void deactivate_swap_slots_cache(void);
47static void reactivate_swap_slots_cache(void);
48
49#define use_swap_slot_cache (swap_slot_cache_active && \
50 swap_slot_cache_enabled && swap_slot_cache_initialized)
51#define SLOTS_CACHE 0x1
52#define SLOTS_CACHE_RET 0x2
53
54static void deactivate_swap_slots_cache(void)
55{
56 mutex_lock(&swap_slots_cache_mutex);
57 swap_slot_cache_active = false;
58 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
59 mutex_unlock(&swap_slots_cache_mutex);
60}
61
62static void reactivate_swap_slots_cache(void)
63{
64 mutex_lock(&swap_slots_cache_mutex);
65 swap_slot_cache_active = true;
66 mutex_unlock(&swap_slots_cache_mutex);
67}
68
69/* Must not be called with cpu hot plug lock */
70void disable_swap_slots_cache_lock(void)
71{
72 mutex_lock(&swap_slots_cache_enable_mutex);
73 swap_slot_cache_enabled = false;
74 if (swap_slot_cache_initialized) {
75 /* serialize with cpu hotplug operations */
76 get_online_cpus();
77 __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
78 put_online_cpus();
79 }
80}
81
82static void __reenable_swap_slots_cache(void)
83{
84 swap_slot_cache_enabled = has_usable_swap();
85}
86
87void reenable_swap_slots_cache_unlock(void)
88{
89 __reenable_swap_slots_cache();
90 mutex_unlock(&swap_slots_cache_enable_mutex);
91}
92
93static bool check_cache_active(void)
94{
95 long pages;
96
97 if (!swap_slot_cache_enabled || !swap_slot_cache_initialized)
98 return false;
99
100 pages = get_nr_swap_pages();
101 if (!swap_slot_cache_active) {
102 if (pages > num_online_cpus() *
103 THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE)
104 reactivate_swap_slots_cache();
105 goto out;
106 }
107
108 /* if global pool of slot caches too low, deactivate cache */
109 if (pages < num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE)
110 deactivate_swap_slots_cache();
111out:
112 return swap_slot_cache_active;
113}
114
115static int alloc_swap_slot_cache(unsigned int cpu)
116{
117 struct swap_slots_cache *cache;
118 swp_entry_t *slots, *slots_ret;
119
120 /*
121 * Do allocation outside swap_slots_cache_mutex
122 * as vzalloc could trigger reclaim and get_swap_page,
123 * which can lock swap_slots_cache_mutex.
124 */
125 slots = vzalloc(sizeof(swp_entry_t) * SWAP_SLOTS_CACHE_SIZE);
126 if (!slots)
127 return -ENOMEM;
128
129 slots_ret = vzalloc(sizeof(swp_entry_t) * SWAP_SLOTS_CACHE_SIZE);
130 if (!slots_ret) {
131 vfree(slots);
132 return -ENOMEM;
133 }
134
135 mutex_lock(&swap_slots_cache_mutex);
136 cache = &per_cpu(swp_slots, cpu);
137 if (cache->slots || cache->slots_ret)
138 /* cache already allocated */
139 goto out;
140 if (!cache->lock_initialized) {
141 mutex_init(&cache->alloc_lock);
142 spin_lock_init(&cache->free_lock);
143 cache->lock_initialized = true;
144 }
145 cache->nr = 0;
146 cache->cur = 0;
147 cache->n_ret = 0;
148 cache->slots = slots;
149 slots = NULL;
150 cache->slots_ret = slots_ret;
151 slots_ret = NULL;
152out:
153 mutex_unlock(&swap_slots_cache_mutex);
154 if (slots)
155 vfree(slots);
156 if (slots_ret)
157 vfree(slots_ret);
158 return 0;
159}
160
161static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
162 bool free_slots)
163{
164 struct swap_slots_cache *cache;
165 swp_entry_t *slots = NULL;
166
167 cache = &per_cpu(swp_slots, cpu);
168 if ((type & SLOTS_CACHE) && cache->slots) {
169 mutex_lock(&cache->alloc_lock);
170 swapcache_free_entries(cache->slots + cache->cur, cache->nr);
171 cache->cur = 0;
172 cache->nr = 0;
173 if (free_slots && cache->slots) {
174 vfree(cache->slots);
175 cache->slots = NULL;
176 }
177 mutex_unlock(&cache->alloc_lock);
178 }
179 if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {
180 spin_lock_irq(&cache->free_lock);
181 swapcache_free_entries(cache->slots_ret, cache->n_ret);
182 cache->n_ret = 0;
183 if (free_slots && cache->slots_ret) {
184 slots = cache->slots_ret;
185 cache->slots_ret = NULL;
186 }
187 spin_unlock_irq(&cache->free_lock);
188 if (slots)
189 vfree(slots);
190 }
191}
192
193static void __drain_swap_slots_cache(unsigned int type)
194{
195 unsigned int cpu;
196
197 /*
198 * This function is called during
199 * 1) swapoff, when we have to make sure no
200 * left over slots are in cache when we remove
201 * a swap device;
202 * 2) disabling of swap slot cache, when we run low
203 * on swap slots when allocating memory and need
204 * to return swap slots to global pool.
205 *
206 * We cannot acquire cpu hot plug lock here as
207 * this function can be invoked in the cpu
208 * hot plug path:
209 * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
210 * -> memory allocation -> direct reclaim -> get_swap_page
211 * -> drain_swap_slots_cache
212 *
213 * Hence the loop over current online cpu below could miss cpu that
214 * is being brought online but not yet marked as online.
215 * That is okay as we do not schedule and run anything on a
216 * cpu before it has been marked online. Hence, we will not
217 * fill any swap slots in slots cache of such cpu.
218 * There are no slots on such cpu that need to be drained.
219 */
220 for_each_online_cpu(cpu)
221 drain_slots_cache_cpu(cpu, type, false);
222}
223
224static int free_slot_cache(unsigned int cpu)
225{
226 mutex_lock(&swap_slots_cache_mutex);
227 drain_slots_cache_cpu(cpu, SLOTS_CACHE | SLOTS_CACHE_RET, true);
228 mutex_unlock(&swap_slots_cache_mutex);
229 return 0;
230}
231
232int enable_swap_slots_cache(void)
233{
234 int ret = 0;
235
236 mutex_lock(&swap_slots_cache_enable_mutex);
237 if (swap_slot_cache_initialized) {
238 __reenable_swap_slots_cache();
239 goto out_unlock;
240 }
241
242 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
243 alloc_swap_slot_cache, free_slot_cache);
244 if (ret < 0)
245 goto out_unlock;
246 swap_slot_cache_initialized = true;
247 __reenable_swap_slots_cache();
248out_unlock:
249 mutex_unlock(&swap_slots_cache_enable_mutex);
250 return 0;
251}
252
253/* called with swap slot cache's alloc lock held */
254static int refill_swap_slots_cache(struct swap_slots_cache *cache)
255{
256 if (!use_swap_slot_cache || cache->nr)
257 return 0;
258
259 cache->cur = 0;
260 if (swap_slot_cache_active)
261 cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE, cache->slots);
262
263 return cache->nr;
264}
265
266int free_swap_slot(swp_entry_t entry)
267{
268 struct swap_slots_cache *cache;
269
Tim Chen67afa382017-02-22 15:45:39 -0800270 cache = &get_cpu_var(swp_slots);
271 if (use_swap_slot_cache && cache->slots_ret) {
272 spin_lock_irq(&cache->free_lock);
273 /* Swap slots cache may be deactivated before acquiring lock */
274 if (!use_swap_slot_cache) {
275 spin_unlock_irq(&cache->free_lock);
276 goto direct_free;
277 }
278 if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {
279 /*
280 * Return slots to global pool.
281 * The current swap_map value is SWAP_HAS_CACHE.
282 * Set it to 0 to indicate it is available for
283 * allocation in global pool
284 */
285 swapcache_free_entries(cache->slots_ret, cache->n_ret);
286 cache->n_ret = 0;
287 }
288 cache->slots_ret[cache->n_ret++] = entry;
289 spin_unlock_irq(&cache->free_lock);
290 } else {
291direct_free:
292 swapcache_free_entries(&entry, 1);
293 }
294 put_cpu_var(swp_slots);
295
296 return 0;
297}
298
299swp_entry_t get_swap_page(void)
300{
301 swp_entry_t entry, *pentry;
302 struct swap_slots_cache *cache;
303
304 /*
305 * Preemption is allowed here, because we may sleep
306 * in refill_swap_slots_cache(). But it is safe, because
307 * accesses to the per-CPU data structure are protected by the
308 * mutex cache->alloc_lock.
309 *
310 * The alloc path here does not touch cache->slots_ret
311 * so cache->free_lock is not taken.
312 */
313 cache = raw_cpu_ptr(&swp_slots);
314
315 entry.val = 0;
316 if (check_cache_active()) {
317 mutex_lock(&cache->alloc_lock);
318 if (cache->slots) {
319repeat:
320 if (cache->nr) {
321 pentry = &cache->slots[cache->cur++];
322 entry = *pentry;
323 pentry->val = 0;
324 cache->nr--;
325 } else {
326 if (refill_swap_slots_cache(cache))
327 goto repeat;
328 }
329 }
330 mutex_unlock(&cache->alloc_lock);
331 if (entry.val)
332 return entry;
333 }
334
335 get_swap_pages(1, &entry);
336
337 return entry;
338}
339
340#endif /* CONFIG_SWAP */