blob: a2d0c68426169e59fb5daaeda397d014bdf72d7b [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/vmscan.c
3 *
4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
5 *
6 * Swap reorganised 29.12.95, Stephen Tweedie.
7 * kswapd added: 7.1.96 sct
8 * Removed kswapd_ctl limits, and swap out as many pages as needed
9 * to bring the system back to freepages.high: 2.4.97, Rik van Riel.
10 * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
11 * Multiqueue VM started 5.8.00, Rik van Riel.
12 */
13
14#include <linux/mm.h>
15#include <linux/module.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090016#include <linux/gfp.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/kernel_stat.h>
18#include <linux/swap.h>
19#include <linux/pagemap.h>
20#include <linux/init.h>
21#include <linux/highmem.h>
Anton Vorontsov70ddf632013-04-29 15:08:31 -070022#include <linux/vmpressure.h>
Andrew Mortone129b5c2006-09-27 01:50:00 -070023#include <linux/vmstat.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070024#include <linux/file.h>
25#include <linux/writeback.h>
26#include <linux/blkdev.h>
27#include <linux/buffer_head.h> /* for try_to_release_page(),
28 buffer_heads_over_limit */
29#include <linux/mm_inline.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070030#include <linux/backing-dev.h>
31#include <linux/rmap.h>
32#include <linux/topology.h>
33#include <linux/cpu.h>
34#include <linux/cpuset.h>
Mel Gorman3e7d3442011-01-13 15:45:56 -080035#include <linux/compaction.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070036#include <linux/notifier.h>
37#include <linux/rwsem.h>
Rafael J. Wysocki248a0302006-03-22 00:09:04 -080038#include <linux/delay.h>
Yasunori Goto3218ae12006-06-27 02:53:33 -070039#include <linux/kthread.h>
Nigel Cunningham7dfb7102006-12-06 20:34:23 -080040#include <linux/freezer.h>
Balbir Singh66e17072008-02-07 00:13:56 -080041#include <linux/memcontrol.h>
Keika Kobayashi873b4772008-07-25 01:48:52 -070042#include <linux/delayacct.h>
Lee Schermerhornaf936a12008-10-18 20:26:53 -070043#include <linux/sysctl.h>
KOSAKI Motohiro929bea72011-04-14 15:22:12 -070044#include <linux/oom.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070045#include <linux/prefetch.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070046
47#include <asm/tlbflush.h>
48#include <asm/div64.h>
49
50#include <linux/swapops.h>
51
Nick Piggin0f8053a2006-03-22 00:08:33 -080052#include "internal.h"
53
Mel Gorman33906bc2010-08-09 17:19:16 -070054#define CREATE_TRACE_POINTS
55#include <trace/events/vmscan.h>
56
Linus Torvalds1da177e2005-04-16 15:20:36 -070057struct scan_control {
Linus Torvalds1da177e2005-04-16 15:20:36 -070058 /* Incremented by the number of inactive pages that were scanned */
59 unsigned long nr_scanned;
60
Rik van Riela79311c2009-01-06 14:40:01 -080061 /* Number of pages freed so far during a call to shrink_zones() */
62 unsigned long nr_reclaimed;
63
KOSAKI Motohiro22fba332009-12-14 17:59:10 -080064 /* How many pages shrink_list() should reclaim */
65 unsigned long nr_to_reclaim;
66
KOSAKI Motohiro7b517552009-12-14 17:59:12 -080067 unsigned long hibernation_mode;
68
Linus Torvalds1da177e2005-04-16 15:20:36 -070069 /* This context's GFP mask */
Al Viro6daa0e22005-10-21 03:18:50 -040070 gfp_t gfp_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
72 int may_writepage;
73
Johannes Weinera6dc60f82009-03-31 15:19:30 -070074 /* Can mapped pages be reclaimed? */
75 int may_unmap;
Christoph Lameterf1fd1062006-01-18 17:42:30 -080076
KOSAKI Motohiro2e2e4252009-04-21 12:24:57 -070077 /* Can pages be swapped as part of reclaim? */
78 int may_swap;
79
Andy Whitcroft5ad333e2007-07-17 04:03:16 -070080 int order;
Balbir Singh66e17072008-02-07 00:13:56 -080081
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -070082 /* Scan (total_size >> priority) pages at once */
83 int priority;
84
KOSAKI Motohiro5f53e762010-05-24 14:32:37 -070085 /*
Johannes Weinerf16015f2012-01-12 17:17:52 -080086 * The memory cgroup that hit its limit and as a result is the
87 * primary target of this reclaim invocation.
88 */
89 struct mem_cgroup *target_mem_cgroup;
Balbir Singh66e17072008-02-07 00:13:56 -080090
KAMEZAWA Hiroyuki327c0e92009-03-31 15:23:31 -070091 /*
92 * Nodemask of nodes allowed by the caller. If NULL, all nodes
93 * are scanned.
94 */
95 nodemask_t *nodemask;
Linus Torvalds1da177e2005-04-16 15:20:36 -070096};
97
Linus Torvalds1da177e2005-04-16 15:20:36 -070098#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
99
100#ifdef ARCH_HAS_PREFETCH
101#define prefetch_prev_lru_page(_page, _base, _field) \
102 do { \
103 if ((_page)->lru.prev != _base) { \
104 struct page *prev; \
105 \
106 prev = lru_to_page(&(_page->lru)); \
107 prefetch(&prev->_field); \
108 } \
109 } while (0)
110#else
111#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
112#endif
113
114#ifdef ARCH_HAS_PREFETCHW
115#define prefetchw_prev_lru_page(_page, _base, _field) \
116 do { \
117 if ((_page)->lru.prev != _base) { \
118 struct page *prev; \
119 \
120 prev = lru_to_page(&(_page->lru)); \
121 prefetchw(&prev->_field); \
122 } \
123 } while (0)
124#else
125#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
126#endif
127
128/*
129 * From 0 .. 100. Higher means more swappy.
130 */
131int vm_swappiness = 60;
Zhang Yanfeib21e0b92013-02-22 16:35:48 -0800132unsigned long vm_total_pages; /* The total number of pages which the VM controls */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700133
134static LIST_HEAD(shrinker_list);
135static DECLARE_RWSEM(shrinker_rwsem);
136
Andrew Mortonc255a452012-07-31 16:43:02 -0700137#ifdef CONFIG_MEMCG
Johannes Weiner89b5fae2012-01-12 17:17:50 -0800138static bool global_reclaim(struct scan_control *sc)
139{
Johannes Weinerf16015f2012-01-12 17:17:52 -0800140 return !sc->target_mem_cgroup;
Johannes Weiner89b5fae2012-01-12 17:17:50 -0800141}
KAMEZAWA Hiroyuki91a45472008-02-07 00:14:29 -0800142#else
Johannes Weiner89b5fae2012-01-12 17:17:50 -0800143static bool global_reclaim(struct scan_control *sc)
144{
145 return true;
146}
KAMEZAWA Hiroyuki91a45472008-02-07 00:14:29 -0800147#endif
148
Hugh Dickins4d7dcca2012-05-29 15:07:08 -0700149static unsigned long get_lru_size(struct lruvec *lruvec, enum lru_list lru)
KOSAKI Motohiroc9f299d2009-01-07 18:08:16 -0800150{
Hugh Dickinsc3c787e2012-05-29 15:06:52 -0700151 if (!mem_cgroup_disabled())
Hugh Dickins4d7dcca2012-05-29 15:07:08 -0700152 return mem_cgroup_get_lru_size(lruvec, lru);
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -0800153
Konstantin Khlebnikov074291f2012-05-29 15:07:00 -0700154 return zone_page_state(lruvec_zone(lruvec), NR_LRU_BASE + lru);
KOSAKI Motohiroc9f299d2009-01-07 18:08:16 -0800155}
156
Linus Torvalds1da177e2005-04-16 15:20:36 -0700157/*
158 * Add a shrinker callback to be called from the vm
159 */
Rusty Russell8e1f9362007-07-17 04:03:17 -0700160void register_shrinker(struct shrinker *shrinker)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161{
Konstantin Khlebnikov83aeead2011-12-08 14:33:54 -0800162 atomic_long_set(&shrinker->nr_in_batch, 0);
Rusty Russell8e1f9362007-07-17 04:03:17 -0700163 down_write(&shrinker_rwsem);
164 list_add_tail(&shrinker->list, &shrinker_list);
165 up_write(&shrinker_rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700166}
Rusty Russell8e1f9362007-07-17 04:03:17 -0700167EXPORT_SYMBOL(register_shrinker);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168
169/*
170 * Remove one
171 */
Rusty Russell8e1f9362007-07-17 04:03:17 -0700172void unregister_shrinker(struct shrinker *shrinker)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173{
174 down_write(&shrinker_rwsem);
175 list_del(&shrinker->list);
176 up_write(&shrinker_rwsem);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177}
Rusty Russell8e1f9362007-07-17 04:03:17 -0700178EXPORT_SYMBOL(unregister_shrinker);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179
Ying Han1495f232011-05-24 17:12:27 -0700180static inline int do_shrinker_shrink(struct shrinker *shrinker,
181 struct shrink_control *sc,
182 unsigned long nr_to_scan)
183{
184 sc->nr_to_scan = nr_to_scan;
185 return (*shrinker->shrink)(shrinker, sc);
186}
187
Linus Torvalds1da177e2005-04-16 15:20:36 -0700188#define SHRINK_BATCH 128
189/*
190 * Call the shrink functions to age shrinkable caches
191 *
192 * Here we assume it costs one seek to replace a lru page and that it also
193 * takes a seek to recreate a cache object. With this in mind we age equal
194 * percentages of the lru and ageable caches. This should balance the seeks
195 * generated by these structures.
196 *
Simon Arlott183ff222007-10-20 01:27:18 +0200197 * If the vm encountered mapped pages on the LRU it increase the pressure on
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198 * slab to avoid swapping.
199 *
200 * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits.
201 *
202 * `lru_pages' represents the number of on-LRU pages in all the zones which
203 * are eligible for the caller's allocation attempt. It is used for balancing
204 * slab reclaim versus page reclaim.
akpm@osdl.orgb15e0902005-06-21 17:14:35 -0700205 *
206 * Returns the number of slab objects which we shrunk.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207 */
Ying Hana09ed5e2011-05-24 17:12:26 -0700208unsigned long shrink_slab(struct shrink_control *shrink,
Ying Han1495f232011-05-24 17:12:27 -0700209 unsigned long nr_pages_scanned,
Ying Hana09ed5e2011-05-24 17:12:26 -0700210 unsigned long lru_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700211{
212 struct shrinker *shrinker;
Andrew Morton69e05942006-03-22 00:08:19 -0800213 unsigned long ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214
Ying Han1495f232011-05-24 17:12:27 -0700215 if (nr_pages_scanned == 0)
216 nr_pages_scanned = SWAP_CLUSTER_MAX;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700217
Minchan Kimf06590b2011-05-24 17:11:11 -0700218 if (!down_read_trylock(&shrinker_rwsem)) {
219 /* Assume we'll be able to shrink next time */
220 ret = 1;
221 goto out;
222 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700223
224 list_for_each_entry(shrinker, &shrinker_list, list) {
225 unsigned long long delta;
Konstantin Khlebnikov635697c2011-12-08 14:33:51 -0800226 long total_scan;
227 long max_pass;
Dave Chinner09576072011-07-08 14:14:34 +1000228 int shrink_ret = 0;
Dave Chinneracf92b42011-07-08 14:14:35 +1000229 long nr;
230 long new_nr;
Dave Chinnere9299f52011-07-08 14:14:37 +1000231 long batch_size = shrinker->batch ? shrinker->batch
232 : SHRINK_BATCH;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233
Konstantin Khlebnikov635697c2011-12-08 14:33:51 -0800234 max_pass = do_shrinker_shrink(shrinker, shrink, 0);
235 if (max_pass <= 0)
236 continue;
237
Dave Chinneracf92b42011-07-08 14:14:35 +1000238 /*
239 * copy the current shrinker scan count into a local variable
240 * and zero it so that other concurrent shrinker invocations
241 * don't also do this scanning work.
242 */
Konstantin Khlebnikov83aeead2011-12-08 14:33:54 -0800243 nr = atomic_long_xchg(&shrinker->nr_in_batch, 0);
Dave Chinneracf92b42011-07-08 14:14:35 +1000244
245 total_scan = nr;
Ying Han1495f232011-05-24 17:12:27 -0700246 delta = (4 * nr_pages_scanned) / shrinker->seeks;
Andrea Arcangeliea164d72005-11-28 13:44:15 -0800247 delta *= max_pass;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248 do_div(delta, lru_pages + 1);
Dave Chinneracf92b42011-07-08 14:14:35 +1000249 total_scan += delta;
250 if (total_scan < 0) {
David Rientjes88c3bd72009-03-31 15:23:29 -0700251 printk(KERN_ERR "shrink_slab: %pF negative objects to "
252 "delete nr=%ld\n",
Dave Chinneracf92b42011-07-08 14:14:35 +1000253 shrinker->shrink, total_scan);
254 total_scan = max_pass;
Andrea Arcangeliea164d72005-11-28 13:44:15 -0800255 }
256
257 /*
Dave Chinner3567b592011-07-08 14:14:36 +1000258 * We need to avoid excessive windup on filesystem shrinkers
259 * due to large numbers of GFP_NOFS allocations causing the
260 * shrinkers to return -1 all the time. This results in a large
261 * nr being built up so when a shrink that can do some work
262 * comes along it empties the entire cache due to nr >>>
263 * max_pass. This is bad for sustaining a working set in
264 * memory.
265 *
266 * Hence only allow the shrinker to scan the entire cache when
267 * a large delta change is calculated directly.
268 */
269 if (delta < max_pass / 4)
270 total_scan = min(total_scan, max_pass / 2);
271
272 /*
Andrea Arcangeliea164d72005-11-28 13:44:15 -0800273 * Avoid risking looping forever due to too large nr value:
274 * never try to free more than twice the estimate number of
275 * freeable entries.
276 */
Dave Chinneracf92b42011-07-08 14:14:35 +1000277 if (total_scan > max_pass * 2)
278 total_scan = max_pass * 2;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279
Dave Chinneracf92b42011-07-08 14:14:35 +1000280 trace_mm_shrink_slab_start(shrinker, shrink, nr,
Dave Chinner09576072011-07-08 14:14:34 +1000281 nr_pages_scanned, lru_pages,
282 max_pass, delta, total_scan);
283
Dave Chinnere9299f52011-07-08 14:14:37 +1000284 while (total_scan >= batch_size) {
akpm@osdl.orgb15e0902005-06-21 17:14:35 -0700285 int nr_before;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286
Ying Han1495f232011-05-24 17:12:27 -0700287 nr_before = do_shrinker_shrink(shrinker, shrink, 0);
288 shrink_ret = do_shrinker_shrink(shrinker, shrink,
Dave Chinnere9299f52011-07-08 14:14:37 +1000289 batch_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700290 if (shrink_ret == -1)
291 break;
akpm@osdl.orgb15e0902005-06-21 17:14:35 -0700292 if (shrink_ret < nr_before)
293 ret += nr_before - shrink_ret;
Dave Chinnere9299f52011-07-08 14:14:37 +1000294 count_vm_events(SLABS_SCANNED, batch_size);
295 total_scan -= batch_size;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700296
297 cond_resched();
298 }
299
Dave Chinneracf92b42011-07-08 14:14:35 +1000300 /*
301 * move the unused scan count back into the shrinker in a
302 * manner that handles concurrent updates. If we exhausted the
303 * scan, there is no need to do an update.
304 */
Konstantin Khlebnikov83aeead2011-12-08 14:33:54 -0800305 if (total_scan > 0)
306 new_nr = atomic_long_add_return(total_scan,
307 &shrinker->nr_in_batch);
308 else
309 new_nr = atomic_long_read(&shrinker->nr_in_batch);
Dave Chinneracf92b42011-07-08 14:14:35 +1000310
311 trace_mm_shrink_slab_end(shrinker, shrink_ret, nr, new_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312 }
313 up_read(&shrinker_rwsem);
Minchan Kimf06590b2011-05-24 17:11:11 -0700314out:
315 cond_resched();
akpm@osdl.orgb15e0902005-06-21 17:14:35 -0700316 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700317}
318
Linus Torvalds1da177e2005-04-16 15:20:36 -0700319static inline int is_page_cache_freeable(struct page *page)
320{
Johannes Weinerceddc3a2009-09-21 17:03:00 -0700321 /*
322 * A freeable page cache page is referenced only by the caller
323 * that isolated the page, the page cache radix tree and
324 * optional buffer heads at page->private.
325 */
Johannes Weineredcf4742009-09-21 17:02:59 -0700326 return page_count(page) - page_has_private(page) == 2;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700327}
328
KOSAKI Motohiro7d3579e2010-10-26 14:21:42 -0700329static int may_write_to_queue(struct backing_dev_info *bdi,
330 struct scan_control *sc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331{
Christoph Lameter930d9152006-01-08 01:00:47 -0800332 if (current->flags & PF_SWAPWRITE)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333 return 1;
334 if (!bdi_write_congested(bdi))
335 return 1;
336 if (bdi == current->backing_dev_info)
337 return 1;
338 return 0;
339}
340
341/*
342 * We detected a synchronous write error writing a page out. Probably
343 * -ENOSPC. We need to propagate that into the address_space for a subsequent
344 * fsync(), msync() or close().
345 *
346 * The tricky part is that after writepage we cannot touch the mapping: nothing
347 * prevents it from being freed up. But we have a ref on the page and once
348 * that page is locked, the mapping is pinned.
349 *
350 * We're allowed to run sleeping lock_page() here because we know the caller has
351 * __GFP_FS.
352 */
353static void handle_write_error(struct address_space *mapping,
354 struct page *page, int error)
355{
Jens Axboe7eaceac2011-03-10 08:52:07 +0100356 lock_page(page);
Guillaume Chazarain3e9f45b2007-05-08 00:23:25 -0700357 if (page_mapping(page) == mapping)
358 mapping_set_error(mapping, error);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700359 unlock_page(page);
360}
361
Christoph Lameter04e62a22006-06-23 02:03:38 -0700362/* possible outcome of pageout() */
363typedef enum {
364 /* failed to write page out, page is locked */
365 PAGE_KEEP,
366 /* move page to the active list, page is locked */
367 PAGE_ACTIVATE,
368 /* page has been sent to the disk successfully, page is unlocked */
369 PAGE_SUCCESS,
370 /* page is clean and locked */
371 PAGE_CLEAN,
372} pageout_t;
373
Linus Torvalds1da177e2005-04-16 15:20:36 -0700374/*
Andrew Morton1742f192006-03-22 00:08:21 -0800375 * pageout is called by shrink_page_list() for each dirty page.
376 * Calls ->writepage().
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377 */
Andy Whitcroftc661b072007-08-22 14:01:26 -0700378static pageout_t pageout(struct page *page, struct address_space *mapping,
KOSAKI Motohiro7d3579e2010-10-26 14:21:42 -0700379 struct scan_control *sc)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380{
381 /*
382 * If the page is dirty, only perform writeback if that write
383 * will be non-blocking. To prevent this allocation from being
384 * stalled by pagecache activity. But note that there may be
385 * stalls if we need to run get_block(). We could test
386 * PagePrivate for that.
387 *
Vincent Li6aceb532009-12-14 17:58:49 -0800388 * If this process is currently in __generic_file_aio_write() against
Linus Torvalds1da177e2005-04-16 15:20:36 -0700389 * this page's queue, we can perform writeback even if that
390 * will block.
391 *
392 * If the page is swapcache, write it back even if that would
393 * block, for some throttling. This happens by accident, because
394 * swap_backing_dev_info is bust: it doesn't reflect the
395 * congestion state of the swapdevs. Easy to fix, if needed.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 */
397 if (!is_page_cache_freeable(page))
398 return PAGE_KEEP;
399 if (!mapping) {
400 /*
401 * Some data journaling orphaned pages can have
402 * page->mapping == NULL while being dirty with clean buffers.
403 */
David Howells266cf652009-04-03 16:42:36 +0100404 if (page_has_private(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700405 if (try_to_free_buffers(page)) {
406 ClearPageDirty(page);
Harvey Harrisond40cee22008-04-30 00:55:07 -0700407 printk("%s: orphaned page\n", __func__);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 return PAGE_CLEAN;
409 }
410 }
411 return PAGE_KEEP;
412 }
413 if (mapping->a_ops->writepage == NULL)
414 return PAGE_ACTIVATE;
Mel Gorman0e093d992010-10-26 14:21:45 -0700415 if (!may_write_to_queue(mapping->backing_dev_info, sc))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 return PAGE_KEEP;
417
418 if (clear_page_dirty_for_io(page)) {
419 int res;
420 struct writeback_control wbc = {
421 .sync_mode = WB_SYNC_NONE,
422 .nr_to_write = SWAP_CLUSTER_MAX,
OGAWA Hirofumi111ebb62006-06-23 02:03:26 -0700423 .range_start = 0,
424 .range_end = LLONG_MAX,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425 .for_reclaim = 1,
426 };
427
428 SetPageReclaim(page);
429 res = mapping->a_ops->writepage(page, &wbc);
430 if (res < 0)
431 handle_write_error(mapping, page, res);
Zach Brown994fc28c2005-12-15 14:28:17 -0800432 if (res == AOP_WRITEPAGE_ACTIVATE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700433 ClearPageReclaim(page);
434 return PAGE_ACTIVATE;
435 }
Andy Whitcroftc661b072007-08-22 14:01:26 -0700436
Linus Torvalds1da177e2005-04-16 15:20:36 -0700437 if (!PageWriteback(page)) {
438 /* synchronous write or broken a_ops? */
439 ClearPageReclaim(page);
440 }
Mel Gorman23b9da52012-05-29 15:06:20 -0700441 trace_mm_vmscan_writepage(page, trace_reclaim_flags(page));
Andrew Mortone129b5c2006-09-27 01:50:00 -0700442 inc_zone_page_state(page, NR_VMSCAN_WRITE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443 return PAGE_SUCCESS;
444 }
445
446 return PAGE_CLEAN;
447}
448
Andrew Mortona649fd92006-10-17 00:09:36 -0700449/*
Nick Piggine2867812008-07-25 19:45:30 -0700450 * Same as remove_mapping, but if the page is removed from the mapping, it
451 * gets returned with a refcount of 0.
Andrew Mortona649fd92006-10-17 00:09:36 -0700452 */
Nick Piggine2867812008-07-25 19:45:30 -0700453static int __remove_mapping(struct address_space *mapping, struct page *page)
Christoph Lameter49d2e9c2006-01-08 01:00:48 -0800454{
Nick Piggin28e4d962006-09-25 23:31:23 -0700455 BUG_ON(!PageLocked(page));
456 BUG_ON(mapping != page_mapping(page));
Christoph Lameter49d2e9c2006-01-08 01:00:48 -0800457
Nick Piggin19fd6232008-07-25 19:45:32 -0700458 spin_lock_irq(&mapping->tree_lock);
Christoph Lameter49d2e9c2006-01-08 01:00:48 -0800459 /*
Nick Piggin0fd0e6b2006-09-27 01:50:02 -0700460 * The non racy check for a busy page.
461 *
462 * Must be careful with the order of the tests. When someone has
463 * a ref to the page, it may be possible that they dirty it then
464 * drop the reference. So if PageDirty is tested before page_count
465 * here, then the following race may occur:
466 *
467 * get_user_pages(&page);
468 * [user mapping goes away]
469 * write_to(page);
470 * !PageDirty(page) [good]
471 * SetPageDirty(page);
472 * put_page(page);
473 * !page_count(page) [good, discard it]
474 *
475 * [oops, our write_to data is lost]
476 *
477 * Reversing the order of the tests ensures such a situation cannot
478 * escape unnoticed. The smp_rmb is needed to ensure the page->flags
479 * load is not satisfied before that of page->_count.
480 *
481 * Note that if SetPageDirty is always performed via set_page_dirty,
482 * and thus under tree_lock, then this ordering is not required.
Christoph Lameter49d2e9c2006-01-08 01:00:48 -0800483 */
Nick Piggine2867812008-07-25 19:45:30 -0700484 if (!page_freeze_refs(page, 2))
Christoph Lameter49d2e9c2006-01-08 01:00:48 -0800485 goto cannot_free;
Nick Piggine2867812008-07-25 19:45:30 -0700486 /* note: atomic_cmpxchg in page_freeze_refs provides the smp_rmb */
487 if (unlikely(PageDirty(page))) {
488 page_unfreeze_refs(page, 2);
Christoph Lameter49d2e9c2006-01-08 01:00:48 -0800489 goto cannot_free;
Nick Piggine2867812008-07-25 19:45:30 -0700490 }
Christoph Lameter49d2e9c2006-01-08 01:00:48 -0800491
492 if (PageSwapCache(page)) {
493 swp_entry_t swap = { .val = page_private(page) };
494 __delete_from_swap_cache(page);
Nick Piggin19fd6232008-07-25 19:45:32 -0700495 spin_unlock_irq(&mapping->tree_lock);
KAMEZAWA Hiroyukicb4b86b2009-06-16 15:32:52 -0700496 swapcache_free(swap, page);
Nick Piggine2867812008-07-25 19:45:30 -0700497 } else {
Linus Torvalds6072d132010-12-01 13:35:19 -0500498 void (*freepage)(struct page *);
499
500 freepage = mapping->a_ops->freepage;
501
Minchan Kime64a7822011-03-22 16:32:44 -0700502 __delete_from_page_cache(page);
Nick Piggin19fd6232008-07-25 19:45:32 -0700503 spin_unlock_irq(&mapping->tree_lock);
Daisuke Nishimurae767e052009-05-28 14:34:28 -0700504 mem_cgroup_uncharge_cache_page(page);
Linus Torvalds6072d132010-12-01 13:35:19 -0500505
506 if (freepage != NULL)
507 freepage(page);
Christoph Lameter49d2e9c2006-01-08 01:00:48 -0800508 }
509
Christoph Lameter49d2e9c2006-01-08 01:00:48 -0800510 return 1;
511
512cannot_free:
Nick Piggin19fd6232008-07-25 19:45:32 -0700513 spin_unlock_irq(&mapping->tree_lock);
Christoph Lameter49d2e9c2006-01-08 01:00:48 -0800514 return 0;
515}
516
Linus Torvalds1da177e2005-04-16 15:20:36 -0700517/*
Nick Piggine2867812008-07-25 19:45:30 -0700518 * Attempt to detach a locked page from its ->mapping. If it is dirty or if
519 * someone else has a ref on the page, abort and return 0. If it was
520 * successfully detached, return 1. Assumes the caller has a single ref on
521 * this page.
522 */
523int remove_mapping(struct address_space *mapping, struct page *page)
524{
525 if (__remove_mapping(mapping, page)) {
526 /*
527 * Unfreezing the refcount with 1 rather than 2 effectively
528 * drops the pagecache ref for us without requiring another
529 * atomic operation.
530 */
531 page_unfreeze_refs(page, 1);
532 return 1;
533 }
534 return 0;
535}
536
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700537/**
538 * putback_lru_page - put previously isolated page onto appropriate LRU list
539 * @page: page to be put back to appropriate lru list
540 *
541 * Add previously isolated @page to appropriate LRU list.
542 * Page may still be unevictable for other reasons.
543 *
544 * lru_lock must not be held, interrupts must be enabled.
545 */
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700546void putback_lru_page(struct page *page)
547{
548 int lru;
549 int active = !!TestClearPageActive(page);
Lee Schermerhornbbfd28e2008-10-18 20:26:40 -0700550 int was_unevictable = PageUnevictable(page);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700551
552 VM_BUG_ON(PageLRU(page));
553
554redo:
555 ClearPageUnevictable(page);
556
Hugh Dickins39b5f292012-10-08 16:33:18 -0700557 if (page_evictable(page)) {
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700558 /*
559 * For evictable pages, we can use the cache.
560 * In event of a race, worst case is we end up with an
561 * unevictable page on [in]active list.
562 * We know how to handle that.
563 */
Johannes Weiner401a8e12009-09-21 17:02:58 -0700564 lru = active + page_lru_base_type(page);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700565 lru_cache_add_lru(page, lru);
566 } else {
567 /*
568 * Put unevictable pages directly on zone's unevictable
569 * list.
570 */
571 lru = LRU_UNEVICTABLE;
572 add_page_to_unevictable_list(page);
Johannes Weiner6a7b9542009-10-26 16:50:00 -0700573 /*
Minchan Kim21ee9f32011-10-31 17:09:28 -0700574 * When racing with an mlock or AS_UNEVICTABLE clearing
575 * (page is unlocked) make sure that if the other thread
576 * does not observe our setting of PG_lru and fails
Hugh Dickins24513262012-01-20 14:34:21 -0800577 * isolation/check_move_unevictable_pages,
Minchan Kim21ee9f32011-10-31 17:09:28 -0700578 * we see PG_mlocked/AS_UNEVICTABLE cleared below and move
Johannes Weiner6a7b9542009-10-26 16:50:00 -0700579 * the page back to the evictable list.
580 *
Minchan Kim21ee9f32011-10-31 17:09:28 -0700581 * The other side is TestClearPageMlocked() or shmem_lock().
Johannes Weiner6a7b9542009-10-26 16:50:00 -0700582 */
583 smp_mb();
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700584 }
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700585
586 /*
587 * page's status can change while we move it among lru. If an evictable
588 * page is on unevictable list, it never be freed. To avoid that,
589 * check after we added it to the list, again.
590 */
Hugh Dickins39b5f292012-10-08 16:33:18 -0700591 if (lru == LRU_UNEVICTABLE && page_evictable(page)) {
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700592 if (!isolate_lru_page(page)) {
593 put_page(page);
594 goto redo;
595 }
596 /* This means someone else dropped this page from LRU
597 * So, it will be freed or putback to LRU again. There is
598 * nothing to do here.
599 */
600 }
601
Lee Schermerhornbbfd28e2008-10-18 20:26:40 -0700602 if (was_unevictable && lru != LRU_UNEVICTABLE)
603 count_vm_event(UNEVICTABLE_PGRESCUED);
604 else if (!was_unevictable && lru == LRU_UNEVICTABLE)
605 count_vm_event(UNEVICTABLE_PGCULLED);
606
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700607 put_page(page); /* drop ref from isolate */
608}
609
Johannes Weinerdfc8d632010-03-05 13:42:19 -0800610enum page_references {
611 PAGEREF_RECLAIM,
612 PAGEREF_RECLAIM_CLEAN,
Johannes Weiner645747462010-03-05 13:42:22 -0800613 PAGEREF_KEEP,
Johannes Weinerdfc8d632010-03-05 13:42:19 -0800614 PAGEREF_ACTIVATE,
615};
616
617static enum page_references page_check_references(struct page *page,
618 struct scan_control *sc)
619{
Johannes Weiner645747462010-03-05 13:42:22 -0800620 int referenced_ptes, referenced_page;
Johannes Weinerdfc8d632010-03-05 13:42:19 -0800621 unsigned long vm_flags;
Johannes Weinerdfc8d632010-03-05 13:42:19 -0800622
Johannes Weinerc3ac9a82012-05-29 15:06:25 -0700623 referenced_ptes = page_referenced(page, 1, sc->target_mem_cgroup,
624 &vm_flags);
Johannes Weiner645747462010-03-05 13:42:22 -0800625 referenced_page = TestClearPageReferenced(page);
Johannes Weinerdfc8d632010-03-05 13:42:19 -0800626
Johannes Weinerdfc8d632010-03-05 13:42:19 -0800627 /*
628 * Mlock lost the isolation race with us. Let try_to_unmap()
629 * move the page to the unevictable list.
630 */
631 if (vm_flags & VM_LOCKED)
632 return PAGEREF_RECLAIM;
633
Johannes Weiner645747462010-03-05 13:42:22 -0800634 if (referenced_ptes) {
Michal Hockoe48982732012-05-29 15:06:45 -0700635 if (PageSwapBacked(page))
Johannes Weiner645747462010-03-05 13:42:22 -0800636 return PAGEREF_ACTIVATE;
637 /*
638 * All mapped pages start out with page table
639 * references from the instantiating fault, so we need
640 * to look twice if a mapped file page is used more
641 * than once.
642 *
643 * Mark it and spare it for another trip around the
644 * inactive list. Another page table reference will
645 * lead to its activation.
646 *
647 * Note: the mark is set for activated pages as well
648 * so that recently deactivated but used pages are
649 * quickly recovered.
650 */
651 SetPageReferenced(page);
652
Konstantin Khlebnikov34dbc672012-01-10 15:06:59 -0800653 if (referenced_page || referenced_ptes > 1)
Johannes Weiner645747462010-03-05 13:42:22 -0800654 return PAGEREF_ACTIVATE;
655
Konstantin Khlebnikovc909e992012-01-10 15:07:03 -0800656 /*
657 * Activate file-backed executable pages after first usage.
658 */
659 if (vm_flags & VM_EXEC)
660 return PAGEREF_ACTIVATE;
661
Johannes Weiner645747462010-03-05 13:42:22 -0800662 return PAGEREF_KEEP;
663 }
Johannes Weinerdfc8d632010-03-05 13:42:19 -0800664
665 /* Reclaim if clean, defer dirty pages to writeback */
KOSAKI Motohiro2e302442010-10-26 14:21:46 -0700666 if (referenced_page && !PageSwapBacked(page))
Johannes Weiner645747462010-03-05 13:42:22 -0800667 return PAGEREF_RECLAIM_CLEAN;
668
669 return PAGEREF_RECLAIM;
Johannes Weinerdfc8d632010-03-05 13:42:19 -0800670}
671
Nick Piggine2867812008-07-25 19:45:30 -0700672/*
Andrew Morton1742f192006-03-22 00:08:21 -0800673 * shrink_page_list() returns the number of reclaimed pages
Linus Torvalds1da177e2005-04-16 15:20:36 -0700674 */
Andrew Morton1742f192006-03-22 00:08:21 -0800675static unsigned long shrink_page_list(struct list_head *page_list,
Konstantin Khlebnikov6a18adb2012-05-29 15:06:59 -0700676 struct zone *zone,
Mel Gormanf84f6e22011-10-31 17:07:51 -0700677 struct scan_control *sc,
Minchan Kim02c6de82012-10-08 16:31:55 -0700678 enum ttu_flags ttu_flags,
Mel Gormand43006d2013-07-03 15:01:50 -0700679 unsigned long *ret_nr_unqueued_dirty,
Minchan Kim02c6de82012-10-08 16:31:55 -0700680 unsigned long *ret_nr_writeback,
681 bool force_reclaim)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700682{
683 LIST_HEAD(ret_pages);
Mel Gormanabe4c3b2010-08-09 17:19:31 -0700684 LIST_HEAD(free_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700685 int pgactivate = 0;
Mel Gormand43006d2013-07-03 15:01:50 -0700686 unsigned long nr_unqueued_dirty = 0;
Mel Gorman0e093d992010-10-26 14:21:45 -0700687 unsigned long nr_dirty = 0;
688 unsigned long nr_congested = 0;
Andrew Morton05ff5132006-03-22 00:08:20 -0800689 unsigned long nr_reclaimed = 0;
Mel Gorman92df3a72011-10-31 17:07:56 -0700690 unsigned long nr_writeback = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700691
692 cond_resched();
693
Tim Chen69980e32012-07-31 16:46:08 -0700694 mem_cgroup_uncharge_start();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700695 while (!list_empty(page_list)) {
696 struct address_space *mapping;
697 struct page *page;
698 int may_enter_fs;
Minchan Kim02c6de82012-10-08 16:31:55 -0700699 enum page_references references = PAGEREF_RECLAIM_CLEAN;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700700
701 cond_resched();
702
703 page = lru_to_page(page_list);
704 list_del(&page->lru);
705
Nick Piggin529ae9a2008-08-02 12:01:03 +0200706 if (!trylock_page(page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 goto keep;
708
Nick Piggin725d7042006-09-25 23:30:55 -0700709 VM_BUG_ON(PageActive(page));
Konstantin Khlebnikov6a18adb2012-05-29 15:06:59 -0700710 VM_BUG_ON(page_zone(page) != zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711
712 sc->nr_scanned++;
Christoph Lameter80e43422006-02-11 17:55:53 -0800713
Hugh Dickins39b5f292012-10-08 16:33:18 -0700714 if (unlikely(!page_evictable(page)))
Nick Pigginb291f002008-10-18 20:26:44 -0700715 goto cull_mlocked;
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700716
Johannes Weinera6dc60f82009-03-31 15:19:30 -0700717 if (!sc->may_unmap && page_mapped(page))
Christoph Lameter80e43422006-02-11 17:55:53 -0800718 goto keep_locked;
719
Linus Torvalds1da177e2005-04-16 15:20:36 -0700720 /* Double the slab pressure for mapped and swapcache pages */
721 if (page_mapped(page) || PageSwapCache(page))
722 sc->nr_scanned++;
723
Andy Whitcroftc661b072007-08-22 14:01:26 -0700724 may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
725 (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
726
Mel Gorman283aba92013-07-03 15:01:51 -0700727 /*
728 * If a page at the tail of the LRU is under writeback, there
729 * are three cases to consider.
730 *
731 * 1) If reclaim is encountering an excessive number of pages
732 * under writeback and this page is both under writeback and
733 * PageReclaim then it indicates that pages are being queued
734 * for IO but are being recycled through the LRU before the
735 * IO can complete. Waiting on the page itself risks an
736 * indefinite stall if it is impossible to writeback the
737 * page due to IO error or disconnected storage so instead
738 * block for HZ/10 or until some IO completes then clear the
739 * ZONE_WRITEBACK flag to recheck if the condition exists.
740 *
741 * 2) Global reclaim encounters a page, memcg encounters a
742 * page that is not marked for immediate reclaim or
743 * the caller does not have __GFP_IO. In this case mark
744 * the page for immediate reclaim and continue scanning.
745 *
746 * __GFP_IO is checked because a loop driver thread might
747 * enter reclaim, and deadlock if it waits on a page for
748 * which it is needed to do the write (loop masks off
749 * __GFP_IO|__GFP_FS for this reason); but more thought
750 * would probably show more reasons.
751 *
752 * Don't require __GFP_FS, since we're not going into the
753 * FS, just waiting on its writeback completion. Worryingly,
754 * ext4 gfs2 and xfs allocate pages with
755 * grab_cache_page_write_begin(,,AOP_FLAG_NOFS), so testing
756 * may_enter_fs here is liable to OOM on them.
757 *
758 * 3) memcg encounters a page that is not already marked
759 * PageReclaim. memcg does not have any dirty pages
760 * throttling so we could easily OOM just because too many
761 * pages are in writeback and there is nothing else to
762 * reclaim. Wait for the writeback to complete.
763 */
Andy Whitcroftc661b072007-08-22 14:01:26 -0700764 if (PageWriteback(page)) {
Mel Gorman283aba92013-07-03 15:01:51 -0700765 /* Case 1 above */
766 if (current_is_kswapd() &&
767 PageReclaim(page) &&
768 zone_is_reclaim_writeback(zone)) {
769 unlock_page(page);
770 congestion_wait(BLK_RW_ASYNC, HZ/10);
771 zone_clear_flag(zone, ZONE_WRITEBACK);
772 goto keep;
773
774 /* Case 2 above */
775 } else if (global_reclaim(sc) ||
Hugh Dickinsc3b94f42012-07-31 16:45:59 -0700776 !PageReclaim(page) || !(sc->gfp_mask & __GFP_IO)) {
777 /*
778 * This is slightly racy - end_page_writeback()
779 * might have just cleared PageReclaim, then
780 * setting PageReclaim here end up interpreted
781 * as PageReadahead - but that does not matter
782 * enough to care. What we do want is for this
783 * page to have PageReclaim set next time memcg
784 * reclaim reaches the tests above, so it will
785 * then wait_on_page_writeback() to avoid OOM;
786 * and it's also appropriate in global reclaim.
787 */
788 SetPageReclaim(page);
Michal Hockoe62e3842012-07-31 16:45:55 -0700789 nr_writeback++;
Mel Gorman283aba92013-07-03 15:01:51 -0700790
Hugh Dickinsc3b94f42012-07-31 16:45:59 -0700791 goto keep_locked;
Mel Gorman283aba92013-07-03 15:01:51 -0700792
793 /* Case 3 above */
794 } else {
795 wait_on_page_writeback(page);
Michal Hockoe62e3842012-07-31 16:45:55 -0700796 }
Andy Whitcroftc661b072007-08-22 14:01:26 -0700797 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700798
Minchan Kim02c6de82012-10-08 16:31:55 -0700799 if (!force_reclaim)
800 references = page_check_references(page, sc);
801
Johannes Weinerdfc8d632010-03-05 13:42:19 -0800802 switch (references) {
803 case PAGEREF_ACTIVATE:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700804 goto activate_locked;
Johannes Weiner645747462010-03-05 13:42:22 -0800805 case PAGEREF_KEEP:
806 goto keep_locked;
Johannes Weinerdfc8d632010-03-05 13:42:19 -0800807 case PAGEREF_RECLAIM:
808 case PAGEREF_RECLAIM_CLEAN:
809 ; /* try to reclaim the page below */
810 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812 /*
813 * Anonymous process memory has backing store?
814 * Try to allocate it some swap space here.
815 */
Nick Pigginb291f002008-10-18 20:26:44 -0700816 if (PageAnon(page) && !PageSwapCache(page)) {
Hugh Dickins63eb6b92008-11-19 15:36:37 -0800817 if (!(sc->gfp_mask & __GFP_IO))
818 goto keep_locked;
Shaohua Li5bc7b8a2013-04-29 15:08:36 -0700819 if (!add_to_swap(page, page_list))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700820 goto activate_locked;
Hugh Dickins63eb6b92008-11-19 15:36:37 -0800821 may_enter_fs = 1;
Nick Pigginb291f002008-10-18 20:26:44 -0700822 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700823
824 mapping = page_mapping(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700825
826 /*
827 * The page is mapped into the page tables of one or more
828 * processes. Try to unmap it here.
829 */
830 if (page_mapped(page) && mapping) {
Minchan Kim02c6de82012-10-08 16:31:55 -0700831 switch (try_to_unmap(page, ttu_flags)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700832 case SWAP_FAIL:
833 goto activate_locked;
834 case SWAP_AGAIN:
835 goto keep_locked;
Nick Pigginb291f002008-10-18 20:26:44 -0700836 case SWAP_MLOCK:
837 goto cull_mlocked;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838 case SWAP_SUCCESS:
839 ; /* try to free the page below */
840 }
841 }
842
843 if (PageDirty(page)) {
Mel Gorman0e093d992010-10-26 14:21:45 -0700844 nr_dirty++;
845
Mel Gormand43006d2013-07-03 15:01:50 -0700846 if (!PageWriteback(page))
847 nr_unqueued_dirty++;
848
Mel Gormanee728862011-10-31 17:07:38 -0700849 /*
850 * Only kswapd can writeback filesystem pages to
Mel Gormand43006d2013-07-03 15:01:50 -0700851 * avoid risk of stack overflow but only writeback
852 * if many dirty pages have been encountered.
Mel Gormanee728862011-10-31 17:07:38 -0700853 */
Mel Gormanf84f6e22011-10-31 17:07:51 -0700854 if (page_is_file_cache(page) &&
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -0700855 (!current_is_kswapd() ||
Mel Gormand43006d2013-07-03 15:01:50 -0700856 !zone_is_reclaim_dirty(zone))) {
Mel Gorman49ea7eb2011-10-31 17:07:59 -0700857 /*
858 * Immediately reclaim when written back.
859 * Similar in principal to deactivate_page()
860 * except we already have the page isolated
861 * and know it's dirty
862 */
863 inc_zone_page_state(page, NR_VMSCAN_IMMEDIATE);
864 SetPageReclaim(page);
865
Mel Gormanee728862011-10-31 17:07:38 -0700866 goto keep_locked;
867 }
868
Johannes Weinerdfc8d632010-03-05 13:42:19 -0800869 if (references == PAGEREF_RECLAIM_CLEAN)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870 goto keep_locked;
Andrew Morton4dd4b922008-03-24 12:29:52 -0700871 if (!may_enter_fs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872 goto keep_locked;
Christoph Lameter52a83632006-02-01 03:05:28 -0800873 if (!sc->may_writepage)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 goto keep_locked;
875
876 /* Page is dirty, try to write it out here */
KOSAKI Motohiro7d3579e2010-10-26 14:21:42 -0700877 switch (pageout(page, mapping, sc)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700878 case PAGE_KEEP:
Mel Gorman0e093d992010-10-26 14:21:45 -0700879 nr_congested++;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700880 goto keep_locked;
881 case PAGE_ACTIVATE:
882 goto activate_locked;
883 case PAGE_SUCCESS:
KOSAKI Motohiro7d3579e2010-10-26 14:21:42 -0700884 if (PageWriteback(page))
Mel Gorman41ac1992012-05-29 15:06:19 -0700885 goto keep;
KOSAKI Motohiro7d3579e2010-10-26 14:21:42 -0700886 if (PageDirty(page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700887 goto keep;
KOSAKI Motohiro7d3579e2010-10-26 14:21:42 -0700888
Linus Torvalds1da177e2005-04-16 15:20:36 -0700889 /*
890 * A synchronous write - probably a ramdisk. Go
891 * ahead and try to reclaim the page.
892 */
Nick Piggin529ae9a2008-08-02 12:01:03 +0200893 if (!trylock_page(page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894 goto keep;
895 if (PageDirty(page) || PageWriteback(page))
896 goto keep_locked;
897 mapping = page_mapping(page);
898 case PAGE_CLEAN:
899 ; /* try to free the page below */
900 }
901 }
902
903 /*
904 * If the page has buffers, try to free the buffer mappings
905 * associated with this page. If we succeed we try to free
906 * the page as well.
907 *
908 * We do this even if the page is PageDirty().
909 * try_to_release_page() does not perform I/O, but it is
910 * possible for a page to have PageDirty set, but it is actually
911 * clean (all its buffers are clean). This happens if the
912 * buffers were written out directly, with submit_bh(). ext3
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700913 * will do this, as well as the blockdev mapping.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700914 * try_to_release_page() will discover that cleanness and will
915 * drop the buffers and mark the page clean - it can be freed.
916 *
917 * Rarely, pages can have buffers and no ->mapping. These are
918 * the pages which were not successfully invalidated in
919 * truncate_complete_page(). We try to drop those buffers here
920 * and if that worked, and the page is no longer mapped into
921 * process address space (page_count == 1) it can be freed.
922 * Otherwise, leave the page on the LRU so it is swappable.
923 */
David Howells266cf652009-04-03 16:42:36 +0100924 if (page_has_private(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700925 if (!try_to_release_page(page, sc->gfp_mask))
926 goto activate_locked;
Nick Piggine2867812008-07-25 19:45:30 -0700927 if (!mapping && page_count(page) == 1) {
928 unlock_page(page);
929 if (put_page_testzero(page))
930 goto free_it;
931 else {
932 /*
933 * rare race with speculative reference.
934 * the speculative reference will free
935 * this page shortly, so we may
936 * increment nr_reclaimed here (and
937 * leave it off the LRU).
938 */
939 nr_reclaimed++;
940 continue;
941 }
942 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700943 }
944
Nick Piggine2867812008-07-25 19:45:30 -0700945 if (!mapping || !__remove_mapping(mapping, page))
Christoph Lameter49d2e9c2006-01-08 01:00:48 -0800946 goto keep_locked;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700947
Nick Piggina978d6f2008-10-18 20:26:58 -0700948 /*
949 * At this point, we have no other references and there is
950 * no way to pick any more up (removed from LRU, removed
951 * from pagecache). Can use non-atomic bitops now (and
952 * we obviously don't have to worry about waking up a process
953 * waiting on the page lock, because there are no references.
954 */
955 __clear_page_locked(page);
Nick Piggine2867812008-07-25 19:45:30 -0700956free_it:
Andrew Morton05ff5132006-03-22 00:08:20 -0800957 nr_reclaimed++;
Mel Gormanabe4c3b2010-08-09 17:19:31 -0700958
959 /*
960 * Is there need to periodically free_page_list? It would
961 * appear not as the counts should be low
962 */
963 list_add(&page->lru, &free_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964 continue;
965
Nick Pigginb291f002008-10-18 20:26:44 -0700966cull_mlocked:
Hugh Dickins63d6c5a2009-01-06 14:39:38 -0800967 if (PageSwapCache(page))
968 try_to_free_swap(page);
Nick Pigginb291f002008-10-18 20:26:44 -0700969 unlock_page(page);
970 putback_lru_page(page);
971 continue;
972
Linus Torvalds1da177e2005-04-16 15:20:36 -0700973activate_locked:
Rik van Riel68a223942008-10-18 20:26:23 -0700974 /* Not a candidate for swapping, so reclaim swap space. */
975 if (PageSwapCache(page) && vm_swap_full())
Hugh Dickinsa2c43ee2009-01-06 14:39:36 -0800976 try_to_free_swap(page);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700977 VM_BUG_ON(PageActive(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700978 SetPageActive(page);
979 pgactivate++;
980keep_locked:
981 unlock_page(page);
982keep:
983 list_add(&page->lru, &ret_pages);
Nick Pigginb291f002008-10-18 20:26:44 -0700984 VM_BUG_ON(PageLRU(page) || PageUnevictable(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700985 }
Mel Gormanabe4c3b2010-08-09 17:19:31 -0700986
Mel Gorman0e093d992010-10-26 14:21:45 -0700987 /*
988 * Tag a zone as congested if all the dirty pages encountered were
989 * backed by a congested BDI. In this case, reclaimers should just
990 * back off and wait for congestion to clear because further reclaim
991 * will encounter the same problem
992 */
Johannes Weiner89b5fae2012-01-12 17:17:50 -0800993 if (nr_dirty && nr_dirty == nr_congested && global_reclaim(sc))
Konstantin Khlebnikov6a18adb2012-05-29 15:06:59 -0700994 zone_set_flag(zone, ZONE_CONGESTED);
Mel Gorman0e093d992010-10-26 14:21:45 -0700995
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -0800996 free_hot_cold_page_list(&free_pages, 1);
Mel Gormanabe4c3b2010-08-09 17:19:31 -0700997
Linus Torvalds1da177e2005-04-16 15:20:36 -0700998 list_splice(&ret_pages, page_list);
Christoph Lameterf8891e52006-06-30 01:55:45 -0700999 count_vm_events(PGACTIVATE, pgactivate);
Tim Chen69980e32012-07-31 16:46:08 -07001000 mem_cgroup_uncharge_end();
Mel Gormand43006d2013-07-03 15:01:50 -07001001 *ret_nr_unqueued_dirty += nr_unqueued_dirty;
Mel Gorman92df3a72011-10-31 17:07:56 -07001002 *ret_nr_writeback += nr_writeback;
Andrew Morton05ff5132006-03-22 00:08:20 -08001003 return nr_reclaimed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001004}
1005
Minchan Kim02c6de82012-10-08 16:31:55 -07001006unsigned long reclaim_clean_pages_from_list(struct zone *zone,
1007 struct list_head *page_list)
1008{
1009 struct scan_control sc = {
1010 .gfp_mask = GFP_KERNEL,
1011 .priority = DEF_PRIORITY,
1012 .may_unmap = 1,
1013 };
1014 unsigned long ret, dummy1, dummy2;
1015 struct page *page, *next;
1016 LIST_HEAD(clean_pages);
1017
1018 list_for_each_entry_safe(page, next, page_list, lru) {
1019 if (page_is_file_cache(page) && !PageDirty(page)) {
1020 ClearPageActive(page);
1021 list_move(&page->lru, &clean_pages);
1022 }
1023 }
1024
1025 ret = shrink_page_list(&clean_pages, zone, &sc,
1026 TTU_UNMAP|TTU_IGNORE_ACCESS,
1027 &dummy1, &dummy2, true);
1028 list_splice(&clean_pages, page_list);
1029 __mod_zone_page_state(zone, NR_ISOLATED_FILE, -ret);
1030 return ret;
1031}
1032
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07001033/*
1034 * Attempt to remove the specified page from its LRU. Only take this page
1035 * if it is of the appropriate PageActive status. Pages which are being
1036 * freed elsewhere are also ignored.
1037 *
1038 * page: page to consider
1039 * mode: one of the LRU isolation modes defined above
1040 *
1041 * returns 0 on success, -ve errno on failure.
1042 */
Konstantin Khlebnikovf3fd4a62012-05-29 15:06:54 -07001043int __isolate_lru_page(struct page *page, isolate_mode_t mode)
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07001044{
1045 int ret = -EINVAL;
1046
1047 /* Only take pages on the LRU. */
1048 if (!PageLRU(page))
1049 return ret;
1050
Minchan Kime46a2872012-10-08 16:33:48 -07001051 /* Compaction should not handle unevictable pages but CMA can do so */
1052 if (PageUnevictable(page) && !(mode & ISOLATE_UNEVICTABLE))
Lee Schermerhorn894bc312008-10-18 20:26:39 -07001053 return ret;
1054
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07001055 ret = -EBUSY;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001056
Mel Gormanc8244932012-01-12 17:19:38 -08001057 /*
1058 * To minimise LRU disruption, the caller can indicate that it only
1059 * wants to isolate pages it will be able to operate on without
1060 * blocking - clean pages for the most part.
1061 *
1062 * ISOLATE_CLEAN means that only clean pages should be isolated. This
1063 * is used by reclaim when it is cannot write to backing storage
1064 *
1065 * ISOLATE_ASYNC_MIGRATE is used to indicate that it only wants to pages
1066 * that it is possible to migrate without blocking
1067 */
1068 if (mode & (ISOLATE_CLEAN|ISOLATE_ASYNC_MIGRATE)) {
1069 /* All the caller can do on PageWriteback is block */
1070 if (PageWriteback(page))
1071 return ret;
1072
1073 if (PageDirty(page)) {
1074 struct address_space *mapping;
1075
1076 /* ISOLATE_CLEAN means only clean pages */
1077 if (mode & ISOLATE_CLEAN)
1078 return ret;
1079
1080 /*
1081 * Only pages without mappings or that have a
1082 * ->migratepage callback are possible to migrate
1083 * without blocking
1084 */
1085 mapping = page_mapping(page);
1086 if (mapping && !mapping->a_ops->migratepage)
1087 return ret;
1088 }
1089 }
Minchan Kim39deaf82011-10-31 17:06:51 -07001090
Minchan Kimf80c0672011-10-31 17:06:55 -07001091 if ((mode & ISOLATE_UNMAPPED) && page_mapped(page))
1092 return ret;
1093
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07001094 if (likely(get_page_unless_zero(page))) {
1095 /*
1096 * Be careful not to clear PageLRU until after we're
1097 * sure the page is not being freed elsewhere -- the
1098 * page release code relies on it.
1099 */
1100 ClearPageLRU(page);
1101 ret = 0;
1102 }
1103
1104 return ret;
1105}
1106
Christoph Lameter49d2e9c2006-01-08 01:00:48 -08001107/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001108 * zone->lru_lock is heavily contended. Some of the functions that
1109 * shrink the lists perform better by taking out a batch of pages
1110 * and working on them outside the LRU lock.
1111 *
1112 * For pagecache intensive workloads, this function is the hottest
1113 * spot in the kernel (apart from copy_*_user functions).
1114 *
1115 * Appropriate locks must be held before calling this function.
1116 *
1117 * @nr_to_scan: The number of pages to look through on the list.
Konstantin Khlebnikov5dc35972012-05-29 15:06:58 -07001118 * @lruvec: The LRU vector to pull pages from.
Linus Torvalds1da177e2005-04-16 15:20:36 -07001119 * @dst: The temp list to put pages on to.
Hugh Dickinsf6260122012-01-12 17:20:06 -08001120 * @nr_scanned: The number of pages that were scanned.
Rik van Rielfe2c2a12012-03-21 16:33:51 -07001121 * @sc: The scan_control struct for this reclaim session
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07001122 * @mode: One of the LRU isolation modes
Konstantin Khlebnikov3cb99452012-05-29 15:06:53 -07001123 * @lru: LRU list id for isolating
Linus Torvalds1da177e2005-04-16 15:20:36 -07001124 *
1125 * returns how many pages were moved onto *@dst.
1126 */
Andrew Morton69e05942006-03-22 00:08:19 -08001127static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
Konstantin Khlebnikov5dc35972012-05-29 15:06:58 -07001128 struct lruvec *lruvec, struct list_head *dst,
Rik van Rielfe2c2a12012-03-21 16:33:51 -07001129 unsigned long *nr_scanned, struct scan_control *sc,
Konstantin Khlebnikov3cb99452012-05-29 15:06:53 -07001130 isolate_mode_t mode, enum lru_list lru)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001131{
Hugh Dickins75b00af2012-05-29 15:07:09 -07001132 struct list_head *src = &lruvec->lists[lru];
Andrew Morton69e05942006-03-22 00:08:19 -08001133 unsigned long nr_taken = 0;
Wu Fengguangc9b02d92006-03-22 00:08:23 -08001134 unsigned long scan;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001135
Wu Fengguangc9b02d92006-03-22 00:08:23 -08001136 for (scan = 0; scan < nr_to_scan && !list_empty(src); scan++) {
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07001137 struct page *page;
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001138 int nr_pages;
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07001139
Linus Torvalds1da177e2005-04-16 15:20:36 -07001140 page = lru_to_page(src);
1141 prefetchw_prev_lru_page(page, src, flags);
1142
Nick Piggin725d7042006-09-25 23:30:55 -07001143 VM_BUG_ON(!PageLRU(page));
Nick Piggin8d438f92006-03-22 00:07:59 -08001144
Konstantin Khlebnikovf3fd4a62012-05-29 15:06:54 -07001145 switch (__isolate_lru_page(page, mode)) {
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07001146 case 0:
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001147 nr_pages = hpage_nr_pages(page);
1148 mem_cgroup_update_lru_size(lruvec, lru, -nr_pages);
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07001149 list_move(&page->lru, dst);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001150 nr_taken += nr_pages;
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07001151 break;
Nick Piggin46453a62006-03-22 00:07:58 -08001152
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07001153 case -EBUSY:
1154 /* else it is being freed elsewhere */
1155 list_move(&page->lru, src);
1156 continue;
1157
1158 default:
1159 BUG();
1160 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001161 }
1162
Hugh Dickinsf6260122012-01-12 17:20:06 -08001163 *nr_scanned = scan;
Hugh Dickins75b00af2012-05-29 15:07:09 -07001164 trace_mm_vmscan_lru_isolate(sc->order, nr_to_scan, scan,
1165 nr_taken, mode, is_file_lru(lru));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001166 return nr_taken;
1167}
1168
Nick Piggin62695a82008-10-18 20:26:09 -07001169/**
1170 * isolate_lru_page - tries to isolate a page from its LRU list
1171 * @page: page to isolate from its LRU list
1172 *
1173 * Isolates a @page from an LRU list, clears PageLRU and adjusts the
1174 * vmstat statistic corresponding to whatever LRU list the page was on.
1175 *
1176 * Returns 0 if the page was removed from an LRU list.
1177 * Returns -EBUSY if the page was not on an LRU list.
1178 *
1179 * The returned page will have PageLRU() cleared. If it was found on
Lee Schermerhorn894bc312008-10-18 20:26:39 -07001180 * the active list, it will have PageActive set. If it was found on
1181 * the unevictable list, it will have the PageUnevictable bit set. That flag
1182 * may need to be cleared by the caller before letting the page go.
Nick Piggin62695a82008-10-18 20:26:09 -07001183 *
1184 * The vmstat statistic corresponding to the list on which the page was
1185 * found will be decremented.
1186 *
1187 * Restrictions:
1188 * (1) Must be called with an elevated refcount on the page. This is a
1189 * fundamentnal difference from isolate_lru_pages (which is called
1190 * without a stable reference).
1191 * (2) the lru_lock must not be held.
1192 * (3) interrupts must be enabled.
1193 */
1194int isolate_lru_page(struct page *page)
1195{
1196 int ret = -EBUSY;
1197
Konstantin Khlebnikov0c917312011-05-24 17:12:21 -07001198 VM_BUG_ON(!page_count(page));
1199
Nick Piggin62695a82008-10-18 20:26:09 -07001200 if (PageLRU(page)) {
1201 struct zone *zone = page_zone(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001202 struct lruvec *lruvec;
Nick Piggin62695a82008-10-18 20:26:09 -07001203
1204 spin_lock_irq(&zone->lru_lock);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001205 lruvec = mem_cgroup_page_lruvec(page, zone);
Konstantin Khlebnikov0c917312011-05-24 17:12:21 -07001206 if (PageLRU(page)) {
Lee Schermerhorn894bc312008-10-18 20:26:39 -07001207 int lru = page_lru(page);
Konstantin Khlebnikov0c917312011-05-24 17:12:21 -07001208 get_page(page);
Nick Piggin62695a82008-10-18 20:26:09 -07001209 ClearPageLRU(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001210 del_page_from_lru_list(page, lruvec, lru);
1211 ret = 0;
Nick Piggin62695a82008-10-18 20:26:09 -07001212 }
1213 spin_unlock_irq(&zone->lru_lock);
1214 }
1215 return ret;
1216}
1217
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07001218/*
Fengguang Wud37dd5d2012-12-18 14:23:28 -08001219 * A direct reclaimer may isolate SWAP_CLUSTER_MAX pages from the LRU list and
1220 * then get resheduled. When there are massive number of tasks doing page
1221 * allocation, such sleeping direct reclaimers may keep piling up on each CPU,
1222 * the LRU list will go small and be scanned faster than necessary, leading to
1223 * unnecessary swapping, thrashing and OOM.
Rik van Riel35cd7812009-09-21 17:01:38 -07001224 */
1225static int too_many_isolated(struct zone *zone, int file,
1226 struct scan_control *sc)
1227{
1228 unsigned long inactive, isolated;
1229
1230 if (current_is_kswapd())
1231 return 0;
1232
Johannes Weiner89b5fae2012-01-12 17:17:50 -08001233 if (!global_reclaim(sc))
Rik van Riel35cd7812009-09-21 17:01:38 -07001234 return 0;
1235
1236 if (file) {
1237 inactive = zone_page_state(zone, NR_INACTIVE_FILE);
1238 isolated = zone_page_state(zone, NR_ISOLATED_FILE);
1239 } else {
1240 inactive = zone_page_state(zone, NR_INACTIVE_ANON);
1241 isolated = zone_page_state(zone, NR_ISOLATED_ANON);
1242 }
1243
Fengguang Wu3cf23842012-12-18 14:23:31 -08001244 /*
1245 * GFP_NOIO/GFP_NOFS callers are allowed to isolate more pages, so they
1246 * won't get blocked by normal direct-reclaimers, forming a circular
1247 * deadlock.
1248 */
1249 if ((sc->gfp_mask & GFP_IOFS) == GFP_IOFS)
1250 inactive >>= 3;
1251
Rik van Riel35cd7812009-09-21 17:01:38 -07001252 return isolated > inactive;
1253}
1254
Mel Gorman66635622010-08-09 17:19:30 -07001255static noinline_for_stack void
Hugh Dickins75b00af2012-05-29 15:07:09 -07001256putback_inactive_pages(struct lruvec *lruvec, struct list_head *page_list)
Mel Gorman66635622010-08-09 17:19:30 -07001257{
Konstantin Khlebnikov27ac81d2012-05-29 15:07:00 -07001258 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
1259 struct zone *zone = lruvec_zone(lruvec);
Hugh Dickins3f797682012-01-12 17:20:07 -08001260 LIST_HEAD(pages_to_free);
Mel Gorman66635622010-08-09 17:19:30 -07001261
Mel Gorman66635622010-08-09 17:19:30 -07001262 /*
1263 * Put back any unfreeable pages.
1264 */
Mel Gorman66635622010-08-09 17:19:30 -07001265 while (!list_empty(page_list)) {
Hugh Dickins3f797682012-01-12 17:20:07 -08001266 struct page *page = lru_to_page(page_list);
Mel Gorman66635622010-08-09 17:19:30 -07001267 int lru;
Hugh Dickins3f797682012-01-12 17:20:07 -08001268
Mel Gorman66635622010-08-09 17:19:30 -07001269 VM_BUG_ON(PageLRU(page));
1270 list_del(&page->lru);
Hugh Dickins39b5f292012-10-08 16:33:18 -07001271 if (unlikely(!page_evictable(page))) {
Mel Gorman66635622010-08-09 17:19:30 -07001272 spin_unlock_irq(&zone->lru_lock);
1273 putback_lru_page(page);
1274 spin_lock_irq(&zone->lru_lock);
1275 continue;
1276 }
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001277
1278 lruvec = mem_cgroup_page_lruvec(page, zone);
1279
Linus Torvalds7a608572011-01-17 14:42:19 -08001280 SetPageLRU(page);
Mel Gorman66635622010-08-09 17:19:30 -07001281 lru = page_lru(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001282 add_page_to_lru_list(page, lruvec, lru);
1283
Mel Gorman66635622010-08-09 17:19:30 -07001284 if (is_active_lru(lru)) {
1285 int file = is_file_lru(lru);
Rik van Riel9992af12011-01-13 15:47:13 -08001286 int numpages = hpage_nr_pages(page);
1287 reclaim_stat->recent_rotated[file] += numpages;
Mel Gorman66635622010-08-09 17:19:30 -07001288 }
Hugh Dickins2bcf8872012-01-12 17:19:56 -08001289 if (put_page_testzero(page)) {
1290 __ClearPageLRU(page);
1291 __ClearPageActive(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001292 del_page_from_lru_list(page, lruvec, lru);
Hugh Dickins2bcf8872012-01-12 17:19:56 -08001293
1294 if (unlikely(PageCompound(page))) {
1295 spin_unlock_irq(&zone->lru_lock);
1296 (*get_compound_page_dtor(page))(page);
1297 spin_lock_irq(&zone->lru_lock);
1298 } else
1299 list_add(&page->lru, &pages_to_free);
Mel Gorman66635622010-08-09 17:19:30 -07001300 }
1301 }
Mel Gorman66635622010-08-09 17:19:30 -07001302
Hugh Dickins3f797682012-01-12 17:20:07 -08001303 /*
1304 * To save our caller's stack, now use input list for pages to free.
1305 */
1306 list_splice(&pages_to_free, page_list);
Mel Gorman66635622010-08-09 17:19:30 -07001307}
1308
1309/*
Andrew Morton1742f192006-03-22 00:08:21 -08001310 * shrink_inactive_list() is a helper for shrink_zone(). It returns the number
1311 * of reclaimed pages
Linus Torvalds1da177e2005-04-16 15:20:36 -07001312 */
Mel Gorman66635622010-08-09 17:19:30 -07001313static noinline_for_stack unsigned long
Konstantin Khlebnikov1a93be02012-05-29 15:07:01 -07001314shrink_inactive_list(unsigned long nr_to_scan, struct lruvec *lruvec,
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07001315 struct scan_control *sc, enum lru_list lru)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001316{
1317 LIST_HEAD(page_list);
KOSAKI Motohiroe247dbc2010-08-09 17:19:28 -07001318 unsigned long nr_scanned;
Andrew Morton05ff5132006-03-22 00:08:20 -08001319 unsigned long nr_reclaimed = 0;
KOSAKI Motohiroe247dbc2010-08-09 17:19:28 -07001320 unsigned long nr_taken;
Mel Gorman92df3a72011-10-31 17:07:56 -07001321 unsigned long nr_dirty = 0;
1322 unsigned long nr_writeback = 0;
Konstantin Khlebnikovf3fd4a62012-05-29 15:06:54 -07001323 isolate_mode_t isolate_mode = 0;
Konstantin Khlebnikov3cb99452012-05-29 15:06:53 -07001324 int file = is_file_lru(lru);
Konstantin Khlebnikov1a93be02012-05-29 15:07:01 -07001325 struct zone *zone = lruvec_zone(lruvec);
1326 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
KOSAKI Motohiro78dc5832009-06-16 15:31:40 -07001327
Rik van Riel35cd7812009-09-21 17:01:38 -07001328 while (unlikely(too_many_isolated(zone, file, sc))) {
KOSAKI Motohiro58355c72009-10-26 16:49:35 -07001329 congestion_wait(BLK_RW_ASYNC, HZ/10);
Rik van Riel35cd7812009-09-21 17:01:38 -07001330
1331 /* We are about to die and free our memory. Return now. */
1332 if (fatal_signal_pending(current))
1333 return SWAP_CLUSTER_MAX;
1334 }
1335
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336 lru_add_drain();
Minchan Kimf80c0672011-10-31 17:06:55 -07001337
1338 if (!sc->may_unmap)
Hillf Danton61317282012-03-21 16:33:48 -07001339 isolate_mode |= ISOLATE_UNMAPPED;
Minchan Kimf80c0672011-10-31 17:06:55 -07001340 if (!sc->may_writepage)
Hillf Danton61317282012-03-21 16:33:48 -07001341 isolate_mode |= ISOLATE_CLEAN;
Minchan Kimf80c0672011-10-31 17:06:55 -07001342
Linus Torvalds1da177e2005-04-16 15:20:36 -07001343 spin_lock_irq(&zone->lru_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001344
Konstantin Khlebnikov5dc35972012-05-29 15:06:58 -07001345 nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &page_list,
1346 &nr_scanned, sc, isolate_mode, lru);
Konstantin Khlebnikov95d918f2012-05-29 15:06:59 -07001347
1348 __mod_zone_page_state(zone, NR_LRU_BASE + lru, -nr_taken);
1349 __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken);
1350
Johannes Weiner89b5fae2012-01-12 17:17:50 -08001351 if (global_reclaim(sc)) {
KOSAKI Motohiroe247dbc2010-08-09 17:19:28 -07001352 zone->pages_scanned += nr_scanned;
KOSAKI Motohirob35ea172009-09-21 17:01:36 -07001353 if (current_is_kswapd())
Hugh Dickins75b00af2012-05-29 15:07:09 -07001354 __count_zone_vm_events(PGSCAN_KSWAPD, zone, nr_scanned);
KOSAKI Motohiroe247dbc2010-08-09 17:19:28 -07001355 else
Hugh Dickins75b00af2012-05-29 15:07:09 -07001356 __count_zone_vm_events(PGSCAN_DIRECT, zone, nr_scanned);
KOSAKI Motohiroe247dbc2010-08-09 17:19:28 -07001357 }
Hillf Dantond563c052012-03-21 16:34:02 -07001358 spin_unlock_irq(&zone->lru_lock);
KOSAKI Motohiroa7312862009-09-21 17:01:37 -07001359
Hillf Dantond563c052012-03-21 16:34:02 -07001360 if (nr_taken == 0)
Mel Gorman66635622010-08-09 17:19:30 -07001361 return 0;
KOSAKI Motohiroe247dbc2010-08-09 17:19:28 -07001362
Minchan Kim02c6de82012-10-08 16:31:55 -07001363 nr_reclaimed = shrink_page_list(&page_list, zone, sc, TTU_UNMAP,
1364 &nr_dirty, &nr_writeback, false);
KOSAKI Motohiroe247dbc2010-08-09 17:19:28 -07001365
Hugh Dickins3f797682012-01-12 17:20:07 -08001366 spin_lock_irq(&zone->lru_lock);
1367
Konstantin Khlebnikov95d918f2012-05-29 15:06:59 -07001368 reclaim_stat->recent_scanned[file] += nr_taken;
Hillf Dantond563c052012-03-21 16:34:02 -07001369
Ying Han904249a2012-04-25 16:01:48 -07001370 if (global_reclaim(sc)) {
1371 if (current_is_kswapd())
1372 __count_zone_vm_events(PGSTEAL_KSWAPD, zone,
1373 nr_reclaimed);
1374 else
1375 __count_zone_vm_events(PGSTEAL_DIRECT, zone,
1376 nr_reclaimed);
1377 }
KOSAKI Motohiroe247dbc2010-08-09 17:19:28 -07001378
Konstantin Khlebnikov27ac81d2012-05-29 15:07:00 -07001379 putback_inactive_pages(lruvec, &page_list);
Hugh Dickins3f797682012-01-12 17:20:07 -08001380
Konstantin Khlebnikov95d918f2012-05-29 15:06:59 -07001381 __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
Hugh Dickins3f797682012-01-12 17:20:07 -08001382
1383 spin_unlock_irq(&zone->lru_lock);
1384
1385 free_hot_cold_page_list(&page_list, 1);
Mel Gormane11da5b2010-10-26 14:21:40 -07001386
Mel Gorman92df3a72011-10-31 17:07:56 -07001387 /*
1388 * If reclaim is isolating dirty pages under writeback, it implies
1389 * that the long-lived page allocation rate is exceeding the page
1390 * laundering rate. Either the global limits are not being effective
1391 * at throttling processes due to the page distribution throughout
1392 * zones or there is heavy usage of a slow backing device. The
1393 * only option is to throttle from reclaim context which is not ideal
1394 * as there is no guarantee the dirtying process is throttled in the
1395 * same way balance_dirty_pages() manages.
1396 *
1397 * This scales the number of dirty pages that must be under writeback
1398 * before throttling depending on priority. It is a simple backoff
1399 * function that has the most effect in the range DEF_PRIORITY to
1400 * DEF_PRIORITY-2 which is the priority reclaim is considered to be
1401 * in trouble and reclaim is considered to be in trouble.
1402 *
1403 * DEF_PRIORITY 100% isolated pages must be PageWriteback to throttle
1404 * DEF_PRIORITY-1 50% must be PageWriteback
1405 * DEF_PRIORITY-2 25% must be PageWriteback, kswapd in trouble
1406 * ...
1407 * DEF_PRIORITY-6 For SWAP_CLUSTER_MAX isolated pages, throttle if any
1408 * isolated page is PageWriteback
1409 */
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07001410 if (nr_writeback && nr_writeback >=
Mel Gorman283aba92013-07-03 15:01:51 -07001411 (nr_taken >> (DEF_PRIORITY - sc->priority))) {
Mel Gorman92df3a72011-10-31 17:07:56 -07001412 wait_iff_congested(zone, BLK_RW_ASYNC, HZ/10);
Mel Gorman283aba92013-07-03 15:01:51 -07001413 zone_set_flag(zone, ZONE_WRITEBACK);
1414 }
Mel Gorman92df3a72011-10-31 17:07:56 -07001415
Mel Gormand43006d2013-07-03 15:01:50 -07001416 /*
1417 * Similarly, if many dirty pages are encountered that are not
1418 * currently being written then flag that kswapd should start
1419 * writing back pages.
1420 */
1421 if (global_reclaim(sc) && nr_dirty &&
1422 nr_dirty >= (nr_taken >> (DEF_PRIORITY - sc->priority)))
1423 zone_set_flag(zone, ZONE_TAIL_LRU_DIRTY);
1424
Mel Gormane11da5b2010-10-26 14:21:40 -07001425 trace_mm_vmscan_lru_shrink_inactive(zone->zone_pgdat->node_id,
1426 zone_idx(zone),
1427 nr_scanned, nr_reclaimed,
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07001428 sc->priority,
Mel Gorman23b9da52012-05-29 15:06:20 -07001429 trace_shrink_flags(file));
Andrew Morton05ff5132006-03-22 00:08:20 -08001430 return nr_reclaimed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001431}
1432
Martin Bligh3bb1a8522006-10-28 10:38:24 -07001433/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001434 * This moves pages from the active list to the inactive list.
1435 *
1436 * We move them the other way if the page is referenced by one or more
1437 * processes, from rmap.
1438 *
1439 * If the pages are mostly unmapped, the processing is fast and it is
1440 * appropriate to hold zone->lru_lock across the whole operation. But if
1441 * the pages are mapped, the processing is slow (page_referenced()) so we
1442 * should drop zone->lru_lock around each page. It's impossible to balance
1443 * this, so instead we remove the pages from the LRU while processing them.
1444 * It is safe to rely on PG_active against the non-LRU pages in here because
1445 * nobody will play with that bit on a non-LRU page.
1446 *
1447 * The downside is that we have to touch page->_count against each page.
1448 * But we had to alter page->flags anyway.
1449 */
KAMEZAWA Hiroyuki1cfb4192008-02-07 00:14:37 -08001450
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001451static void move_active_pages_to_lru(struct lruvec *lruvec,
Wu Fengguang3eb41402009-06-16 15:33:13 -07001452 struct list_head *list,
Hugh Dickins2bcf8872012-01-12 17:19:56 -08001453 struct list_head *pages_to_free,
Wu Fengguang3eb41402009-06-16 15:33:13 -07001454 enum lru_list lru)
1455{
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001456 struct zone *zone = lruvec_zone(lruvec);
Wu Fengguang3eb41402009-06-16 15:33:13 -07001457 unsigned long pgmoved = 0;
Wu Fengguang3eb41402009-06-16 15:33:13 -07001458 struct page *page;
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001459 int nr_pages;
Wu Fengguang3eb41402009-06-16 15:33:13 -07001460
Wu Fengguang3eb41402009-06-16 15:33:13 -07001461 while (!list_empty(list)) {
1462 page = lru_to_page(list);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001463 lruvec = mem_cgroup_page_lruvec(page, zone);
Wu Fengguang3eb41402009-06-16 15:33:13 -07001464
1465 VM_BUG_ON(PageLRU(page));
1466 SetPageLRU(page);
1467
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001468 nr_pages = hpage_nr_pages(page);
1469 mem_cgroup_update_lru_size(lruvec, lru, nr_pages);
Johannes Weiner925b7672012-01-12 17:18:15 -08001470 list_move(&page->lru, &lruvec->lists[lru]);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001471 pgmoved += nr_pages;
Wu Fengguang3eb41402009-06-16 15:33:13 -07001472
Hugh Dickins2bcf8872012-01-12 17:19:56 -08001473 if (put_page_testzero(page)) {
1474 __ClearPageLRU(page);
1475 __ClearPageActive(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001476 del_page_from_lru_list(page, lruvec, lru);
Hugh Dickins2bcf8872012-01-12 17:19:56 -08001477
1478 if (unlikely(PageCompound(page))) {
1479 spin_unlock_irq(&zone->lru_lock);
1480 (*get_compound_page_dtor(page))(page);
1481 spin_lock_irq(&zone->lru_lock);
1482 } else
1483 list_add(&page->lru, pages_to_free);
Wu Fengguang3eb41402009-06-16 15:33:13 -07001484 }
1485 }
1486 __mod_zone_page_state(zone, NR_LRU_BASE + lru, pgmoved);
1487 if (!is_active_lru(lru))
1488 __count_vm_events(PGDEACTIVATE, pgmoved);
1489}
KAMEZAWA Hiroyuki1cfb4192008-02-07 00:14:37 -08001490
Hugh Dickinsf6260122012-01-12 17:20:06 -08001491static void shrink_active_list(unsigned long nr_to_scan,
Konstantin Khlebnikov1a93be02012-05-29 15:07:01 -07001492 struct lruvec *lruvec,
Johannes Weinerf16015f2012-01-12 17:17:52 -08001493 struct scan_control *sc,
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07001494 enum lru_list lru)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001495{
KOSAKI Motohiro44c241f2009-09-21 17:01:35 -07001496 unsigned long nr_taken;
Hugh Dickinsf6260122012-01-12 17:20:06 -08001497 unsigned long nr_scanned;
Wu Fengguang6fe6b7e2009-06-16 15:33:05 -07001498 unsigned long vm_flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001499 LIST_HEAD(l_hold); /* The pages which were snipped off */
Wu Fengguang8cab4752009-06-16 15:33:12 -07001500 LIST_HEAD(l_active);
Christoph Lameterb69408e2008-10-18 20:26:14 -07001501 LIST_HEAD(l_inactive);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001502 struct page *page;
Konstantin Khlebnikov1a93be02012-05-29 15:07:01 -07001503 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
KOSAKI Motohiro44c241f2009-09-21 17:01:35 -07001504 unsigned long nr_rotated = 0;
Konstantin Khlebnikovf3fd4a62012-05-29 15:06:54 -07001505 isolate_mode_t isolate_mode = 0;
Konstantin Khlebnikov3cb99452012-05-29 15:06:53 -07001506 int file = is_file_lru(lru);
Konstantin Khlebnikov1a93be02012-05-29 15:07:01 -07001507 struct zone *zone = lruvec_zone(lruvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508
1509 lru_add_drain();
Minchan Kimf80c0672011-10-31 17:06:55 -07001510
1511 if (!sc->may_unmap)
Hillf Danton61317282012-03-21 16:33:48 -07001512 isolate_mode |= ISOLATE_UNMAPPED;
Minchan Kimf80c0672011-10-31 17:06:55 -07001513 if (!sc->may_writepage)
Hillf Danton61317282012-03-21 16:33:48 -07001514 isolate_mode |= ISOLATE_CLEAN;
Minchan Kimf80c0672011-10-31 17:06:55 -07001515
Linus Torvalds1da177e2005-04-16 15:20:36 -07001516 spin_lock_irq(&zone->lru_lock);
Johannes Weiner925b7672012-01-12 17:18:15 -08001517
Konstantin Khlebnikov5dc35972012-05-29 15:06:58 -07001518 nr_taken = isolate_lru_pages(nr_to_scan, lruvec, &l_hold,
1519 &nr_scanned, sc, isolate_mode, lru);
Johannes Weiner89b5fae2012-01-12 17:17:50 -08001520 if (global_reclaim(sc))
Hugh Dickinsf6260122012-01-12 17:20:06 -08001521 zone->pages_scanned += nr_scanned;
Johannes Weiner89b5fae2012-01-12 17:17:50 -08001522
Johannes Weinerb7c46d12009-09-21 17:02:56 -07001523 reclaim_stat->recent_scanned[file] += nr_taken;
KAMEZAWA Hiroyuki1cfb4192008-02-07 00:14:37 -08001524
Hugh Dickinsf6260122012-01-12 17:20:06 -08001525 __count_zone_vm_events(PGREFILL, zone, nr_scanned);
Konstantin Khlebnikov3cb99452012-05-29 15:06:53 -07001526 __mod_zone_page_state(zone, NR_LRU_BASE + lru, -nr_taken);
KOSAKI Motohiroa7312862009-09-21 17:01:37 -07001527 __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, nr_taken);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001528 spin_unlock_irq(&zone->lru_lock);
1529
Linus Torvalds1da177e2005-04-16 15:20:36 -07001530 while (!list_empty(&l_hold)) {
1531 cond_resched();
1532 page = lru_to_page(&l_hold);
1533 list_del(&page->lru);
Rik van Riel7e9cd482008-10-18 20:26:35 -07001534
Hugh Dickins39b5f292012-10-08 16:33:18 -07001535 if (unlikely(!page_evictable(page))) {
Lee Schermerhorn894bc312008-10-18 20:26:39 -07001536 putback_lru_page(page);
1537 continue;
1538 }
1539
Mel Gormancc715d92012-03-21 16:34:00 -07001540 if (unlikely(buffer_heads_over_limit)) {
1541 if (page_has_private(page) && trylock_page(page)) {
1542 if (page_has_private(page))
1543 try_to_release_page(page, 0);
1544 unlock_page(page);
1545 }
1546 }
1547
Johannes Weinerc3ac9a82012-05-29 15:06:25 -07001548 if (page_referenced(page, 0, sc->target_mem_cgroup,
1549 &vm_flags)) {
Rik van Riel9992af12011-01-13 15:47:13 -08001550 nr_rotated += hpage_nr_pages(page);
Wu Fengguang8cab4752009-06-16 15:33:12 -07001551 /*
1552 * Identify referenced, file-backed active pages and
1553 * give them one more trip around the active list. So
1554 * that executable code get better chances to stay in
1555 * memory under moderate memory pressure. Anon pages
1556 * are not likely to be evicted by use-once streaming
1557 * IO, plus JVM can create lots of anon VM_EXEC pages,
1558 * so we ignore them here.
1559 */
Wu Fengguang41e20982009-10-26 16:49:53 -07001560 if ((vm_flags & VM_EXEC) && page_is_file_cache(page)) {
Wu Fengguang8cab4752009-06-16 15:33:12 -07001561 list_add(&page->lru, &l_active);
1562 continue;
1563 }
1564 }
Rik van Riel7e9cd482008-10-18 20:26:35 -07001565
KOSAKI Motohiro5205e562009-09-21 17:01:44 -07001566 ClearPageActive(page); /* we are de-activating */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001567 list_add(&page->lru, &l_inactive);
1568 }
1569
Andrew Mortonb5557492009-01-06 14:40:13 -08001570 /*
Wu Fengguang8cab4752009-06-16 15:33:12 -07001571 * Move pages back to the lru list.
Andrew Mortonb5557492009-01-06 14:40:13 -08001572 */
Johannes Weiner2a1dc502008-12-01 03:00:35 +01001573 spin_lock_irq(&zone->lru_lock);
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001574 /*
Wu Fengguang8cab4752009-06-16 15:33:12 -07001575 * Count referenced pages from currently used mappings as rotated,
1576 * even though only some of them are actually re-activated. This
1577 * helps balance scan pressure between file and anonymous pages in
1578 * get_scan_ratio.
Rik van Riel7e9cd482008-10-18 20:26:35 -07001579 */
Johannes Weinerb7c46d12009-09-21 17:02:56 -07001580 reclaim_stat->recent_rotated[file] += nr_rotated;
Rik van Riel556adec2008-10-18 20:26:34 -07001581
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001582 move_active_pages_to_lru(lruvec, &l_active, &l_hold, lru);
1583 move_active_pages_to_lru(lruvec, &l_inactive, &l_hold, lru - LRU_ACTIVE);
KOSAKI Motohiroa7312862009-09-21 17:01:37 -07001584 __mod_zone_page_state(zone, NR_ISOLATED_ANON + file, -nr_taken);
Christoph Lameterf8891e52006-06-30 01:55:45 -07001585 spin_unlock_irq(&zone->lru_lock);
Hugh Dickins2bcf8872012-01-12 17:19:56 -08001586
1587 free_hot_cold_page_list(&l_hold, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001588}
1589
Minchan Kim74e3f3c2010-10-26 14:21:31 -07001590#ifdef CONFIG_SWAP
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001591static int inactive_anon_is_low_global(struct zone *zone)
KOSAKI Motohirof89eb902009-01-07 18:08:14 -08001592{
1593 unsigned long active, inactive;
1594
1595 active = zone_page_state(zone, NR_ACTIVE_ANON);
1596 inactive = zone_page_state(zone, NR_INACTIVE_ANON);
1597
1598 if (inactive * zone->inactive_ratio < active)
1599 return 1;
1600
1601 return 0;
1602}
1603
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001604/**
1605 * inactive_anon_is_low - check if anonymous pages need to be deactivated
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07001606 * @lruvec: LRU vector to check
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001607 *
1608 * Returns true if the zone does not have enough inactive anon pages,
1609 * meaning some active anon pages need to be deactivated.
1610 */
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07001611static int inactive_anon_is_low(struct lruvec *lruvec)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001612{
Minchan Kim74e3f3c2010-10-26 14:21:31 -07001613 /*
1614 * If we don't have swap space, anonymous page deactivation
1615 * is pointless.
1616 */
1617 if (!total_swap_pages)
1618 return 0;
1619
Hugh Dickinsc3c787e2012-05-29 15:06:52 -07001620 if (!mem_cgroup_disabled())
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07001621 return mem_cgroup_inactive_anon_is_low(lruvec);
Johannes Weinerf16015f2012-01-12 17:17:52 -08001622
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07001623 return inactive_anon_is_low_global(lruvec_zone(lruvec));
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001624}
Minchan Kim74e3f3c2010-10-26 14:21:31 -07001625#else
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07001626static inline int inactive_anon_is_low(struct lruvec *lruvec)
Minchan Kim74e3f3c2010-10-26 14:21:31 -07001627{
1628 return 0;
1629}
1630#endif
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001631
Rik van Riel56e49d22009-06-16 15:32:28 -07001632/**
1633 * inactive_file_is_low - check if file pages need to be deactivated
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07001634 * @lruvec: LRU vector to check
Rik van Riel56e49d22009-06-16 15:32:28 -07001635 *
1636 * When the system is doing streaming IO, memory pressure here
1637 * ensures that active file pages get deactivated, until more
1638 * than half of the file pages are on the inactive list.
1639 *
1640 * Once we get to that situation, protect the system's working
1641 * set from being evicted by disabling active file page aging.
1642 *
1643 * This uses a different ratio than the anonymous pages, because
1644 * the page cache uses a use-once replacement algorithm.
1645 */
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07001646static int inactive_file_is_low(struct lruvec *lruvec)
Rik van Riel56e49d22009-06-16 15:32:28 -07001647{
Johannes Weinere3790142013-02-22 16:35:19 -08001648 unsigned long inactive;
1649 unsigned long active;
Rik van Riel56e49d22009-06-16 15:32:28 -07001650
Johannes Weinere3790142013-02-22 16:35:19 -08001651 inactive = get_lru_size(lruvec, LRU_INACTIVE_FILE);
1652 active = get_lru_size(lruvec, LRU_ACTIVE_FILE);
1653
1654 return active > inactive;
Rik van Riel56e49d22009-06-16 15:32:28 -07001655}
1656
Hugh Dickins75b00af2012-05-29 15:07:09 -07001657static int inactive_list_is_low(struct lruvec *lruvec, enum lru_list lru)
Rik van Rielb39415b2009-12-14 17:59:48 -08001658{
Hugh Dickins75b00af2012-05-29 15:07:09 -07001659 if (is_file_lru(lru))
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07001660 return inactive_file_is_low(lruvec);
Rik van Rielb39415b2009-12-14 17:59:48 -08001661 else
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07001662 return inactive_anon_is_low(lruvec);
Rik van Rielb39415b2009-12-14 17:59:48 -08001663}
1664
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001665static unsigned long shrink_list(enum lru_list lru, unsigned long nr_to_scan,
Konstantin Khlebnikov1a93be02012-05-29 15:07:01 -07001666 struct lruvec *lruvec, struct scan_control *sc)
Christoph Lameterb69408e2008-10-18 20:26:14 -07001667{
Rik van Rielb39415b2009-12-14 17:59:48 -08001668 if (is_active_lru(lru)) {
Hugh Dickins75b00af2012-05-29 15:07:09 -07001669 if (inactive_list_is_low(lruvec, lru))
Konstantin Khlebnikov1a93be02012-05-29 15:07:01 -07001670 shrink_active_list(nr_to_scan, lruvec, sc, lru);
Rik van Riel556adec2008-10-18 20:26:34 -07001671 return 0;
1672 }
1673
Konstantin Khlebnikov1a93be02012-05-29 15:07:01 -07001674 return shrink_inactive_list(nr_to_scan, lruvec, sc, lru);
Christoph Lameterb69408e2008-10-18 20:26:14 -07001675}
1676
Konstantin Khlebnikov3d58ab52012-05-29 15:06:57 -07001677static int vmscan_swappiness(struct scan_control *sc)
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07001678{
Johannes Weiner89b5fae2012-01-12 17:17:50 -08001679 if (global_reclaim(sc))
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07001680 return vm_swappiness;
Konstantin Khlebnikov3d58ab52012-05-29 15:06:57 -07001681 return mem_cgroup_swappiness(sc->target_mem_cgroup);
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07001682}
1683
Johannes Weiner9a265112013-02-22 16:32:17 -08001684enum scan_balance {
1685 SCAN_EQUAL,
1686 SCAN_FRACT,
1687 SCAN_ANON,
1688 SCAN_FILE,
1689};
1690
Linus Torvalds1da177e2005-04-16 15:20:36 -07001691/*
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001692 * Determine how aggressively the anon and file LRU lists should be
1693 * scanned. The relative value of each set of LRU lists is determined
1694 * by looking at the fraction of the pages scanned we did rotate back
1695 * onto the active list instead of evict.
1696 *
Wanpeng Libe7bd592012-06-14 20:41:02 +08001697 * nr[0] = anon inactive pages to scan; nr[1] = anon active pages to scan
1698 * nr[2] = file inactive pages to scan; nr[3] = file active pages to scan
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001699 */
Konstantin Khlebnikov90126372012-05-29 15:07:01 -07001700static void get_scan_count(struct lruvec *lruvec, struct scan_control *sc,
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07001701 unsigned long *nr)
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001702{
Konstantin Khlebnikov90126372012-05-29 15:07:01 -07001703 struct zone_reclaim_stat *reclaim_stat = &lruvec->reclaim_stat;
Johannes Weiner9a265112013-02-22 16:32:17 -08001704 u64 fraction[2];
1705 u64 denominator = 0; /* gcc */
Konstantin Khlebnikov90126372012-05-29 15:07:01 -07001706 struct zone *zone = lruvec_zone(lruvec);
Johannes Weiner9a265112013-02-22 16:32:17 -08001707 unsigned long anon_prio, file_prio;
1708 enum scan_balance scan_balance;
1709 unsigned long anon, file, free;
1710 bool force_scan = false;
1711 unsigned long ap, fp;
1712 enum lru_list lru;
KAMEZAWA Hiroyuki246e87a2011-05-26 16:25:34 -07001713
Johannes Weinerf11c0ca2011-10-31 17:07:27 -07001714 /*
1715 * If the zone or memcg is small, nr[l] can be 0. This
1716 * results in no scanning on this priority and a potential
1717 * priority drop. Global direct reclaim can go to the next
1718 * zone and tends to have no problems. Global kswapd is for
1719 * zone balancing and it needs to scan a minimum amount. When
1720 * reclaiming for a memcg, a priority drop can cause high
1721 * latencies, so it's better to scan a minimum amount there as
1722 * well.
1723 */
Konstantin Khlebnikov90126372012-05-29 15:07:01 -07001724 if (current_is_kswapd() && zone->all_unreclaimable)
Johannes Weinera4d3e9e2011-09-14 16:21:52 -07001725 force_scan = true;
Johannes Weiner89b5fae2012-01-12 17:17:50 -08001726 if (!global_reclaim(sc))
Johannes Weinera4d3e9e2011-09-14 16:21:52 -07001727 force_scan = true;
Shaohua Li76a33fc2010-05-24 14:32:36 -07001728
1729 /* If we have no swap space, do not bother scanning anon pages. */
Shaohua Liec8acf22013-02-22 16:34:38 -08001730 if (!sc->may_swap || (get_nr_swap_pages() <= 0)) {
Johannes Weiner9a265112013-02-22 16:32:17 -08001731 scan_balance = SCAN_FILE;
Shaohua Li76a33fc2010-05-24 14:32:36 -07001732 goto out;
1733 }
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001734
Johannes Weiner10316b32013-02-22 16:32:14 -08001735 /*
1736 * Global reclaim will swap to prevent OOM even with no
1737 * swappiness, but memcg users want to use this knob to
1738 * disable swapping for individual groups completely when
1739 * using the memory controller's swap limit feature would be
1740 * too expensive.
1741 */
1742 if (!global_reclaim(sc) && !vmscan_swappiness(sc)) {
Johannes Weiner9a265112013-02-22 16:32:17 -08001743 scan_balance = SCAN_FILE;
Johannes Weiner10316b32013-02-22 16:32:14 -08001744 goto out;
1745 }
1746
1747 /*
1748 * Do not apply any pressure balancing cleverness when the
1749 * system is close to OOM, scan both anon and file equally
1750 * (unless the swappiness setting disagrees with swapping).
1751 */
1752 if (!sc->priority && vmscan_swappiness(sc)) {
Johannes Weiner9a265112013-02-22 16:32:17 -08001753 scan_balance = SCAN_EQUAL;
Johannes Weiner10316b32013-02-22 16:32:14 -08001754 goto out;
1755 }
1756
Hugh Dickins4d7dcca2012-05-29 15:07:08 -07001757 anon = get_lru_size(lruvec, LRU_ACTIVE_ANON) +
1758 get_lru_size(lruvec, LRU_INACTIVE_ANON);
1759 file = get_lru_size(lruvec, LRU_ACTIVE_FILE) +
1760 get_lru_size(lruvec, LRU_INACTIVE_FILE);
Johannes Weinera4d3e9e2011-09-14 16:21:52 -07001761
Johannes Weiner11d16c22013-02-22 16:32:15 -08001762 /*
1763 * If it's foreseeable that reclaiming the file cache won't be
1764 * enough to get the zone back into a desirable shape, we have
1765 * to swap. Better start now and leave the - probably heavily
1766 * thrashing - remaining file pages alone.
1767 */
Johannes Weiner89b5fae2012-01-12 17:17:50 -08001768 if (global_reclaim(sc)) {
Johannes Weiner11d16c22013-02-22 16:32:15 -08001769 free = zone_page_state(zone, NR_FREE_PAGES);
Konstantin Khlebnikov90126372012-05-29 15:07:01 -07001770 if (unlikely(file + free <= high_wmark_pages(zone))) {
Johannes Weiner9a265112013-02-22 16:32:17 -08001771 scan_balance = SCAN_ANON;
Shaohua Li76a33fc2010-05-24 14:32:36 -07001772 goto out;
KOSAKI Motohiroeeee9a82009-01-07 18:08:17 -08001773 }
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001774 }
1775
1776 /*
Johannes Weiner7c5bd702013-02-22 16:32:10 -08001777 * There is enough inactive page cache, do not reclaim
1778 * anything from the anonymous working set right now.
1779 */
1780 if (!inactive_file_is_low(lruvec)) {
Johannes Weiner9a265112013-02-22 16:32:17 -08001781 scan_balance = SCAN_FILE;
Johannes Weiner7c5bd702013-02-22 16:32:10 -08001782 goto out;
1783 }
1784
Johannes Weiner9a265112013-02-22 16:32:17 -08001785 scan_balance = SCAN_FRACT;
1786
Johannes Weiner7c5bd702013-02-22 16:32:10 -08001787 /*
KOSAKI Motohiro58c37f62010-08-09 17:19:51 -07001788 * With swappiness at 100, anonymous and file have the same priority.
1789 * This scanning priority is essentially the inverse of IO cost.
1790 */
Konstantin Khlebnikov3d58ab52012-05-29 15:06:57 -07001791 anon_prio = vmscan_swappiness(sc);
Hugh Dickins75b00af2012-05-29 15:07:09 -07001792 file_prio = 200 - anon_prio;
KOSAKI Motohiro58c37f62010-08-09 17:19:51 -07001793
1794 /*
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001795 * OK, so we have swap space and a fair amount of page cache
1796 * pages. We use the recently rotated / recently scanned
1797 * ratios to determine how valuable each cache is.
1798 *
1799 * Because workloads change over time (and to avoid overflow)
1800 * we keep these statistics as a floating average, which ends
1801 * up weighing recent references more than old ones.
1802 *
1803 * anon in [0], file in [1]
1804 */
Konstantin Khlebnikov90126372012-05-29 15:07:01 -07001805 spin_lock_irq(&zone->lru_lock);
KOSAKI Motohiro6e901572009-01-07 18:08:15 -08001806 if (unlikely(reclaim_stat->recent_scanned[0] > anon / 4)) {
KOSAKI Motohiro6e901572009-01-07 18:08:15 -08001807 reclaim_stat->recent_scanned[0] /= 2;
1808 reclaim_stat->recent_rotated[0] /= 2;
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001809 }
1810
KOSAKI Motohiro6e901572009-01-07 18:08:15 -08001811 if (unlikely(reclaim_stat->recent_scanned[1] > file / 4)) {
KOSAKI Motohiro6e901572009-01-07 18:08:15 -08001812 reclaim_stat->recent_scanned[1] /= 2;
1813 reclaim_stat->recent_rotated[1] /= 2;
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001814 }
1815
1816 /*
Rik van Riel00d80892008-11-19 15:36:44 -08001817 * The amount of pressure on anon vs file pages is inversely
1818 * proportional to the fraction of recently scanned pages on
1819 * each list that were recently referenced and in active use.
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001820 */
Satoru Moriyafe350042012-05-29 15:06:47 -07001821 ap = anon_prio * (reclaim_stat->recent_scanned[0] + 1);
KOSAKI Motohiro6e901572009-01-07 18:08:15 -08001822 ap /= reclaim_stat->recent_rotated[0] + 1;
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001823
Satoru Moriyafe350042012-05-29 15:06:47 -07001824 fp = file_prio * (reclaim_stat->recent_scanned[1] + 1);
KOSAKI Motohiro6e901572009-01-07 18:08:15 -08001825 fp /= reclaim_stat->recent_rotated[1] + 1;
Konstantin Khlebnikov90126372012-05-29 15:07:01 -07001826 spin_unlock_irq(&zone->lru_lock);
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001827
Shaohua Li76a33fc2010-05-24 14:32:36 -07001828 fraction[0] = ap;
1829 fraction[1] = fp;
1830 denominator = ap + fp + 1;
1831out:
Hugh Dickins41113042012-01-12 17:20:01 -08001832 for_each_evictable_lru(lru) {
1833 int file = is_file_lru(lru);
Johannes Weinerd778df52013-02-22 16:32:12 -08001834 unsigned long size;
Shaohua Li76a33fc2010-05-24 14:32:36 -07001835 unsigned long scan;
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001836
Johannes Weinerd778df52013-02-22 16:32:12 -08001837 size = get_lru_size(lruvec, lru);
Johannes Weiner10316b32013-02-22 16:32:14 -08001838 scan = size >> sc->priority;
Johannes Weiner9a265112013-02-22 16:32:17 -08001839
Johannes Weiner10316b32013-02-22 16:32:14 -08001840 if (!scan && force_scan)
1841 scan = min(size, SWAP_CLUSTER_MAX);
Johannes Weiner9a265112013-02-22 16:32:17 -08001842
1843 switch (scan_balance) {
1844 case SCAN_EQUAL:
1845 /* Scan lists relative to size */
1846 break;
1847 case SCAN_FRACT:
1848 /*
1849 * Scan types proportional to swappiness and
1850 * their relative recent reclaim efficiency.
1851 */
1852 scan = div64_u64(scan * fraction[file], denominator);
1853 break;
1854 case SCAN_FILE:
1855 case SCAN_ANON:
1856 /* Scan one type exclusively */
1857 if ((scan_balance == SCAN_FILE) != file)
1858 scan = 0;
1859 break;
1860 default:
1861 /* Look ma, no brain */
1862 BUG();
1863 }
Hugh Dickins41113042012-01-12 17:20:01 -08001864 nr[lru] = scan;
Shaohua Li76a33fc2010-05-24 14:32:36 -07001865 }
Wu Fengguang6e08a362009-06-16 15:32:29 -07001866}
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001867
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08001868/*
1869 * This is a basic per-zone page freer. Used by both kswapd and direct reclaim.
1870 */
1871static void shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc)
1872{
1873 unsigned long nr[NR_LRU_LISTS];
Mel Gormane82e0562013-07-03 15:01:44 -07001874 unsigned long targets[NR_LRU_LISTS];
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08001875 unsigned long nr_to_scan;
1876 enum lru_list lru;
1877 unsigned long nr_reclaimed = 0;
1878 unsigned long nr_to_reclaim = sc->nr_to_reclaim;
1879 struct blk_plug plug;
Mel Gormane82e0562013-07-03 15:01:44 -07001880 bool scan_adjusted = false;
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08001881
1882 get_scan_count(lruvec, sc, nr);
1883
Mel Gormane82e0562013-07-03 15:01:44 -07001884 /* Record the original scan target for proportional adjustments later */
1885 memcpy(targets, nr, sizeof(nr));
1886
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08001887 blk_start_plug(&plug);
1888 while (nr[LRU_INACTIVE_ANON] || nr[LRU_ACTIVE_FILE] ||
1889 nr[LRU_INACTIVE_FILE]) {
Mel Gormane82e0562013-07-03 15:01:44 -07001890 unsigned long nr_anon, nr_file, percentage;
1891 unsigned long nr_scanned;
1892
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08001893 for_each_evictable_lru(lru) {
1894 if (nr[lru]) {
1895 nr_to_scan = min(nr[lru], SWAP_CLUSTER_MAX);
1896 nr[lru] -= nr_to_scan;
1897
1898 nr_reclaimed += shrink_list(lru, nr_to_scan,
1899 lruvec, sc);
1900 }
1901 }
Mel Gormane82e0562013-07-03 15:01:44 -07001902
1903 if (nr_reclaimed < nr_to_reclaim || scan_adjusted)
1904 continue;
1905
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08001906 /*
Mel Gormane82e0562013-07-03 15:01:44 -07001907 * For global direct reclaim, reclaim only the number of pages
1908 * requested. Less care is taken to scan proportionally as it
1909 * is more important to minimise direct reclaim stall latency
1910 * than it is to properly age the LRU lists.
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08001911 */
Mel Gormane82e0562013-07-03 15:01:44 -07001912 if (global_reclaim(sc) && !current_is_kswapd())
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08001913 break;
Mel Gormane82e0562013-07-03 15:01:44 -07001914
1915 /*
1916 * For kswapd and memcg, reclaim at least the number of pages
1917 * requested. Ensure that the anon and file LRUs shrink
1918 * proportionally what was requested by get_scan_count(). We
1919 * stop reclaiming one LRU and reduce the amount scanning
1920 * proportional to the original scan target.
1921 */
1922 nr_file = nr[LRU_INACTIVE_FILE] + nr[LRU_ACTIVE_FILE];
1923 nr_anon = nr[LRU_INACTIVE_ANON] + nr[LRU_ACTIVE_ANON];
1924
1925 if (nr_file > nr_anon) {
1926 unsigned long scan_target = targets[LRU_INACTIVE_ANON] +
1927 targets[LRU_ACTIVE_ANON] + 1;
1928 lru = LRU_BASE;
1929 percentage = nr_anon * 100 / scan_target;
1930 } else {
1931 unsigned long scan_target = targets[LRU_INACTIVE_FILE] +
1932 targets[LRU_ACTIVE_FILE] + 1;
1933 lru = LRU_FILE;
1934 percentage = nr_file * 100 / scan_target;
1935 }
1936
1937 /* Stop scanning the smaller of the LRU */
1938 nr[lru] = 0;
1939 nr[lru + LRU_ACTIVE] = 0;
1940
1941 /*
1942 * Recalculate the other LRU scan count based on its original
1943 * scan target and the percentage scanning already complete
1944 */
1945 lru = (lru == LRU_FILE) ? LRU_BASE : LRU_FILE;
1946 nr_scanned = targets[lru] - nr[lru];
1947 nr[lru] = targets[lru] * (100 - percentage) / 100;
1948 nr[lru] -= min(nr[lru], nr_scanned);
1949
1950 lru += LRU_ACTIVE;
1951 nr_scanned = targets[lru] - nr[lru];
1952 nr[lru] = targets[lru] * (100 - percentage) / 100;
1953 nr[lru] -= min(nr[lru], nr_scanned);
1954
1955 scan_adjusted = true;
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08001956 }
1957 blk_finish_plug(&plug);
1958 sc->nr_reclaimed += nr_reclaimed;
1959
1960 /*
1961 * Even if we did not try to evict anon pages at all, we want to
1962 * rebalance the anon lru active/inactive ratio.
1963 */
1964 if (inactive_anon_is_low(lruvec))
1965 shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
1966 sc, LRU_ACTIVE_ANON);
1967
1968 throttle_vm_writeout(sc->gfp_mask);
1969}
1970
Mel Gorman23b9da52012-05-29 15:06:20 -07001971/* Use reclaim/compaction for costly allocs or under memory pressure */
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07001972static bool in_reclaim_compaction(struct scan_control *sc)
Mel Gorman23b9da52012-05-29 15:06:20 -07001973{
Kirill A. Shutemovd84da3f2012-12-11 16:00:31 -08001974 if (IS_ENABLED(CONFIG_COMPACTION) && sc->order &&
Mel Gorman23b9da52012-05-29 15:06:20 -07001975 (sc->order > PAGE_ALLOC_COSTLY_ORDER ||
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07001976 sc->priority < DEF_PRIORITY - 2))
Mel Gorman23b9da52012-05-29 15:06:20 -07001977 return true;
1978
1979 return false;
1980}
1981
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001982/*
Mel Gorman23b9da52012-05-29 15:06:20 -07001983 * Reclaim/compaction is used for high-order allocation requests. It reclaims
1984 * order-0 pages before compacting the zone. should_continue_reclaim() returns
1985 * true if more pages should be reclaimed such that when the page allocator
1986 * calls try_to_compact_zone() that it will have enough free pages to succeed.
1987 * It will give up earlier than that if there is difficulty reclaiming pages.
Mel Gorman3e7d3442011-01-13 15:45:56 -08001988 */
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08001989static inline bool should_continue_reclaim(struct zone *zone,
Mel Gorman3e7d3442011-01-13 15:45:56 -08001990 unsigned long nr_reclaimed,
1991 unsigned long nr_scanned,
1992 struct scan_control *sc)
1993{
1994 unsigned long pages_for_compaction;
1995 unsigned long inactive_lru_pages;
1996
1997 /* If not in reclaim/compaction mode, stop */
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07001998 if (!in_reclaim_compaction(sc))
Mel Gorman3e7d3442011-01-13 15:45:56 -08001999 return false;
2000
Mel Gorman28765922011-02-25 14:44:20 -08002001 /* Consider stopping depending on scan and reclaim activity */
2002 if (sc->gfp_mask & __GFP_REPEAT) {
2003 /*
2004 * For __GFP_REPEAT allocations, stop reclaiming if the
2005 * full LRU list has been scanned and we are still failing
2006 * to reclaim pages. This full LRU scan is potentially
2007 * expensive but a __GFP_REPEAT caller really wants to succeed
2008 */
2009 if (!nr_reclaimed && !nr_scanned)
2010 return false;
2011 } else {
2012 /*
2013 * For non-__GFP_REPEAT allocations which can presumably
2014 * fail without consequence, stop if we failed to reclaim
2015 * any pages from the last SWAP_CLUSTER_MAX number of
2016 * pages that were scanned. This will return to the
2017 * caller faster at the risk reclaim/compaction and
2018 * the resulting allocation attempt fails
2019 */
2020 if (!nr_reclaimed)
2021 return false;
2022 }
Mel Gorman3e7d3442011-01-13 15:45:56 -08002023
2024 /*
2025 * If we have not reclaimed enough pages for compaction and the
2026 * inactive lists are large enough, continue reclaiming
2027 */
2028 pages_for_compaction = (2UL << sc->order);
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08002029 inactive_lru_pages = zone_page_state(zone, NR_INACTIVE_FILE);
Shaohua Liec8acf22013-02-22 16:34:38 -08002030 if (get_nr_swap_pages() > 0)
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08002031 inactive_lru_pages += zone_page_state(zone, NR_INACTIVE_ANON);
Mel Gorman3e7d3442011-01-13 15:45:56 -08002032 if (sc->nr_reclaimed < pages_for_compaction &&
2033 inactive_lru_pages > pages_for_compaction)
2034 return true;
2035
2036 /* If compaction would go ahead or the allocation would succeed, stop */
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08002037 switch (compaction_suitable(zone, sc->order)) {
Mel Gorman3e7d3442011-01-13 15:45:56 -08002038 case COMPACT_PARTIAL:
2039 case COMPACT_CONTINUE:
2040 return false;
2041 default:
2042 return true;
2043 }
2044}
2045
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002046static void shrink_zone(struct zone *zone, struct scan_control *sc)
Johannes Weinerf16015f2012-01-12 17:17:52 -08002047{
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08002048 unsigned long nr_reclaimed, nr_scanned;
Johannes Weinerf16015f2012-01-12 17:17:52 -08002049
Johannes Weiner56600482012-01-12 17:17:59 -08002050 do {
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08002051 struct mem_cgroup *root = sc->target_mem_cgroup;
2052 struct mem_cgroup_reclaim_cookie reclaim = {
2053 .zone = zone,
2054 .priority = sc->priority,
2055 };
2056 struct mem_cgroup *memcg;
Johannes Weiner56600482012-01-12 17:17:59 -08002057
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08002058 nr_reclaimed = sc->nr_reclaimed;
2059 nr_scanned = sc->nr_scanned;
Konstantin Khlebnikovf9be23d2012-05-29 15:07:02 -07002060
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08002061 memcg = mem_cgroup_iter(root, NULL, &reclaim);
2062 do {
2063 struct lruvec *lruvec;
2064
2065 lruvec = mem_cgroup_zone_lruvec(zone, memcg);
2066
2067 shrink_lruvec(lruvec, sc);
2068
2069 /*
Michal Hockoa394cb82013-02-22 16:32:30 -08002070 * Direct reclaim and kswapd have to scan all memory
2071 * cgroups to fulfill the overall scan target for the
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08002072 * zone.
Michal Hockoa394cb82013-02-22 16:32:30 -08002073 *
2074 * Limit reclaim, on the other hand, only cares about
2075 * nr_to_reclaim pages to be reclaimed and it will
2076 * retry with decreasing priority if one round over the
2077 * whole hierarchy is not sufficient.
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08002078 */
Michal Hockoa394cb82013-02-22 16:32:30 -08002079 if (!global_reclaim(sc) &&
2080 sc->nr_reclaimed >= sc->nr_to_reclaim) {
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08002081 mem_cgroup_iter_break(root, memcg);
2082 break;
2083 }
2084 memcg = mem_cgroup_iter(root, memcg, &reclaim);
2085 } while (memcg);
Anton Vorontsov70ddf632013-04-29 15:08:31 -07002086
2087 vmpressure(sc->gfp_mask, sc->target_mem_cgroup,
2088 sc->nr_scanned - nr_scanned,
2089 sc->nr_reclaimed - nr_reclaimed);
2090
Johannes Weiner9b4f98c2013-02-22 16:32:19 -08002091 } while (should_continue_reclaim(zone, sc->nr_reclaimed - nr_reclaimed,
2092 sc->nr_scanned - nr_scanned, sc));
Johannes Weinerf16015f2012-01-12 17:17:52 -08002093}
2094
Mel Gormanfe4b1b22012-01-12 17:19:45 -08002095/* Returns true if compaction should go ahead for a high-order request */
2096static inline bool compaction_ready(struct zone *zone, struct scan_control *sc)
2097{
2098 unsigned long balance_gap, watermark;
2099 bool watermark_ok;
2100
2101 /* Do not consider compaction for orders reclaim is meant to satisfy */
2102 if (sc->order <= PAGE_ALLOC_COSTLY_ORDER)
2103 return false;
2104
2105 /*
2106 * Compaction takes time to run and there are potentially other
2107 * callers using the pages just freed. Continue reclaiming until
2108 * there is a buffer of free pages available to give compaction
2109 * a reasonable chance of completing and allocating the page
2110 */
2111 balance_gap = min(low_wmark_pages(zone),
Jiang Liub40da042013-02-22 16:33:52 -08002112 (zone->managed_pages + KSWAPD_ZONE_BALANCE_GAP_RATIO-1) /
Mel Gormanfe4b1b22012-01-12 17:19:45 -08002113 KSWAPD_ZONE_BALANCE_GAP_RATIO);
2114 watermark = high_wmark_pages(zone) + balance_gap + (2UL << sc->order);
2115 watermark_ok = zone_watermark_ok_safe(zone, 0, watermark, 0, 0);
2116
2117 /*
2118 * If compaction is deferred, reclaim up to a point where
2119 * compaction will have a chance of success when re-enabled
2120 */
Rik van Rielaff62242012-03-21 16:33:52 -07002121 if (compaction_deferred(zone, sc->order))
Mel Gormanfe4b1b22012-01-12 17:19:45 -08002122 return watermark_ok;
2123
2124 /* If compaction is not ready to start, keep reclaiming */
2125 if (!compaction_suitable(zone, sc->order))
2126 return false;
2127
2128 return watermark_ok;
2129}
2130
Linus Torvalds1da177e2005-04-16 15:20:36 -07002131/*
2132 * This is the direct reclaim path, for page-allocating processes. We only
2133 * try to reclaim pages from zones which will satisfy the caller's allocation
2134 * request.
2135 *
Mel Gorman41858962009-06-16 15:32:12 -07002136 * We reclaim from a zone even if that zone is over high_wmark_pages(zone).
2137 * Because:
Linus Torvalds1da177e2005-04-16 15:20:36 -07002138 * a) The caller may be trying to free *extra* pages to satisfy a higher-order
2139 * allocation or
Mel Gorman41858962009-06-16 15:32:12 -07002140 * b) The target zone may be at high_wmark_pages(zone) but the lower zones
2141 * must go *over* high_wmark_pages(zone) to satisfy the `incremental min'
2142 * zone defense algorithm.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002143 *
Linus Torvalds1da177e2005-04-16 15:20:36 -07002144 * If a zone is deemed to be full of pinned pages then just give it a light
2145 * scan then give up on it.
Mel Gormane0c23272011-10-31 17:09:33 -07002146 *
2147 * This function returns true if a zone is being reclaimed for a costly
Mel Gormanfe4b1b22012-01-12 17:19:45 -08002148 * high-order allocation and compaction is ready to begin. This indicates to
Mel Gorman0cee34f2012-01-12 17:19:49 -08002149 * the caller that it should consider retrying the allocation instead of
2150 * further reclaim.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002151 */
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002152static bool shrink_zones(struct zonelist *zonelist, struct scan_control *sc)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002153{
Mel Gormandd1a2392008-04-28 02:12:17 -07002154 struct zoneref *z;
Mel Gorman54a6eb52008-04-28 02:12:16 -07002155 struct zone *zone;
Ying Hand149e3b2011-05-26 16:25:27 -07002156 unsigned long nr_soft_reclaimed;
2157 unsigned long nr_soft_scanned;
Mel Gorman0cee34f2012-01-12 17:19:49 -08002158 bool aborted_reclaim = false;
KAMEZAWA Hiroyuki1cfb4192008-02-07 00:14:37 -08002159
Mel Gormancc715d92012-03-21 16:34:00 -07002160 /*
2161 * If the number of buffer_heads in the machine exceeds the maximum
2162 * allowed level, force direct reclaim to scan the highmem zone as
2163 * highmem pages could be pinning lowmem pages storing buffer_heads
2164 */
2165 if (buffer_heads_over_limit)
2166 sc->gfp_mask |= __GFP_HIGHMEM;
2167
Mel Gormand4debc62010-08-09 17:19:29 -07002168 for_each_zone_zonelist_nodemask(zone, z, zonelist,
2169 gfp_zone(sc->gfp_mask), sc->nodemask) {
Con Kolivasf3fe6512006-01-06 00:11:15 -08002170 if (!populated_zone(zone))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002171 continue;
KAMEZAWA Hiroyuki1cfb4192008-02-07 00:14:37 -08002172 /*
2173 * Take care memory controller reclaiming has small influence
2174 * to global LRU.
2175 */
Johannes Weiner89b5fae2012-01-12 17:17:50 -08002176 if (global_reclaim(sc)) {
KAMEZAWA Hiroyuki1cfb4192008-02-07 00:14:37 -08002177 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
2178 continue;
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002179 if (zone->all_unreclaimable &&
2180 sc->priority != DEF_PRIORITY)
KAMEZAWA Hiroyuki1cfb4192008-02-07 00:14:37 -08002181 continue; /* Let kswapd poll it */
Kirill A. Shutemovd84da3f2012-12-11 16:00:31 -08002182 if (IS_ENABLED(CONFIG_COMPACTION)) {
Rik van Riele0887c12011-10-31 17:09:31 -07002183 /*
Mel Gormane0c23272011-10-31 17:09:33 -07002184 * If we already have plenty of memory free for
2185 * compaction in this zone, don't free any more.
2186 * Even though compaction is invoked for any
2187 * non-zero order, only frequent costly order
2188 * reclamation is disruptive enough to become a
Copot Alexandruc7cfa372012-03-21 16:34:10 -07002189 * noticeable problem, like transparent huge
2190 * page allocations.
Rik van Riele0887c12011-10-31 17:09:31 -07002191 */
Mel Gormanfe4b1b22012-01-12 17:19:45 -08002192 if (compaction_ready(zone, sc)) {
Mel Gorman0cee34f2012-01-12 17:19:49 -08002193 aborted_reclaim = true;
Rik van Riele0887c12011-10-31 17:09:31 -07002194 continue;
Mel Gormane0c23272011-10-31 17:09:33 -07002195 }
Rik van Riele0887c12011-10-31 17:09:31 -07002196 }
KAMEZAWA Hiroyukiac34a1a2011-06-27 16:18:12 -07002197 /*
2198 * This steals pages from memory cgroups over softlimit
2199 * and returns the number of reclaimed pages and
2200 * scanned pages. This works for global memory pressure
2201 * and balancing, not for a memcg's limit.
2202 */
2203 nr_soft_scanned = 0;
2204 nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone,
2205 sc->order, sc->gfp_mask,
2206 &nr_soft_scanned);
2207 sc->nr_reclaimed += nr_soft_reclaimed;
2208 sc->nr_scanned += nr_soft_scanned;
2209 /* need some check for avoid more shrink_zone() */
KAMEZAWA Hiroyuki1cfb4192008-02-07 00:14:37 -08002210 }
Nick Piggin408d8542006-09-25 23:31:27 -07002211
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002212 shrink_zone(zone, sc);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002213 }
Mel Gormane0c23272011-10-31 17:09:33 -07002214
Mel Gorman0cee34f2012-01-12 17:19:49 -08002215 return aborted_reclaim;
Minchan Kimd1908362010-09-22 13:05:01 -07002216}
2217
2218static bool zone_reclaimable(struct zone *zone)
2219{
2220 return zone->pages_scanned < zone_reclaimable_pages(zone) * 6;
2221}
2222
KOSAKI Motohiro929bea72011-04-14 15:22:12 -07002223/* All zones in zonelist are unreclaimable? */
Minchan Kimd1908362010-09-22 13:05:01 -07002224static bool all_unreclaimable(struct zonelist *zonelist,
2225 struct scan_control *sc)
2226{
2227 struct zoneref *z;
2228 struct zone *zone;
Minchan Kimd1908362010-09-22 13:05:01 -07002229
2230 for_each_zone_zonelist_nodemask(zone, z, zonelist,
2231 gfp_zone(sc->gfp_mask), sc->nodemask) {
2232 if (!populated_zone(zone))
2233 continue;
2234 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
2235 continue;
KOSAKI Motohiro929bea72011-04-14 15:22:12 -07002236 if (!zone->all_unreclaimable)
2237 return false;
Minchan Kimd1908362010-09-22 13:05:01 -07002238 }
2239
KOSAKI Motohiro929bea72011-04-14 15:22:12 -07002240 return true;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002241}
Rik van Riel4f98a2f2008-10-18 20:26:32 -07002242
Linus Torvalds1da177e2005-04-16 15:20:36 -07002243/*
2244 * This is the main entry point to direct page reclaim.
2245 *
2246 * If a full scan of the inactive list fails to free enough memory then we
2247 * are "out of memory" and something needs to be killed.
2248 *
2249 * If the caller is !__GFP_FS then the probability of a failure is reasonably
2250 * high - the zone may be full of dirty or under-writeback pages, which this
Jens Axboe5b0830c2009-09-23 19:37:09 +02002251 * caller can't do much about. We kick the writeback threads and take explicit
2252 * naps in the hope that some of these pages can be written. But if the
2253 * allocating task holds filesystem locks which prevent writeout this might not
2254 * work, and the allocation attempt will fail.
Nishanth Aravamudana41f24e2008-04-29 00:58:25 -07002255 *
2256 * returns: 0, if no pages reclaimed
2257 * else, the number of pages reclaimed
Linus Torvalds1da177e2005-04-16 15:20:36 -07002258 */
Mel Gormandac1d272008-04-28 02:12:12 -07002259static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
Ying Hana09ed5e2011-05-24 17:12:26 -07002260 struct scan_control *sc,
2261 struct shrink_control *shrink)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002262{
Andrew Morton69e05942006-03-22 00:08:19 -08002263 unsigned long total_scanned = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002264 struct reclaim_state *reclaim_state = current->reclaim_state;
Mel Gormandd1a2392008-04-28 02:12:17 -07002265 struct zoneref *z;
Mel Gorman54a6eb52008-04-28 02:12:16 -07002266 struct zone *zone;
KOSAKI Motohiro22fba332009-12-14 17:59:10 -08002267 unsigned long writeback_threshold;
Mel Gorman0cee34f2012-01-12 17:19:49 -08002268 bool aborted_reclaim;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002269
Keika Kobayashi873b4772008-07-25 01:48:52 -07002270 delayacct_freepages_start();
2271
Johannes Weiner89b5fae2012-01-12 17:17:50 -08002272 if (global_reclaim(sc))
KAMEZAWA Hiroyuki1cfb4192008-02-07 00:14:37 -08002273 count_vm_event(ALLOCSTALL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002274
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002275 do {
Anton Vorontsov70ddf632013-04-29 15:08:31 -07002276 vmpressure_prio(sc->gfp_mask, sc->target_mem_cgroup,
2277 sc->priority);
Balbir Singh66e17072008-02-07 00:13:56 -08002278 sc->nr_scanned = 0;
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002279 aborted_reclaim = shrink_zones(zonelist, sc);
Mel Gormane0c23272011-10-31 17:09:33 -07002280
Balbir Singh66e17072008-02-07 00:13:56 -08002281 /*
2282 * Don't shrink slabs when reclaiming memory from
2283 * over limit cgroups
2284 */
Johannes Weiner89b5fae2012-01-12 17:17:50 -08002285 if (global_reclaim(sc)) {
KOSAKI Motohiroc6a8a8c2010-08-09 17:19:14 -07002286 unsigned long lru_pages = 0;
Mel Gormand4debc62010-08-09 17:19:29 -07002287 for_each_zone_zonelist(zone, z, zonelist,
2288 gfp_zone(sc->gfp_mask)) {
KOSAKI Motohiroc6a8a8c2010-08-09 17:19:14 -07002289 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
2290 continue;
2291
2292 lru_pages += zone_reclaimable_pages(zone);
2293 }
2294
Ying Han1495f232011-05-24 17:12:27 -07002295 shrink_slab(shrink, sc->nr_scanned, lru_pages);
KAMEZAWA Hiroyuki91a45472008-02-07 00:14:29 -08002296 if (reclaim_state) {
Rik van Riela79311c2009-01-06 14:40:01 -08002297 sc->nr_reclaimed += reclaim_state->reclaimed_slab;
KAMEZAWA Hiroyuki91a45472008-02-07 00:14:29 -08002298 reclaim_state->reclaimed_slab = 0;
2299 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002300 }
Balbir Singh66e17072008-02-07 00:13:56 -08002301 total_scanned += sc->nr_scanned;
KOSAKI Motohirobb21c7c2010-06-04 14:15:05 -07002302 if (sc->nr_reclaimed >= sc->nr_to_reclaim)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002303 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002304
2305 /*
Minchan Kim0e50ce32013-02-22 16:35:37 -08002306 * If we're getting trouble reclaiming, start doing
2307 * writepage even in laptop mode.
2308 */
2309 if (sc->priority < DEF_PRIORITY - 2)
2310 sc->may_writepage = 1;
2311
2312 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002313 * Try to write back as many pages as we just scanned. This
2314 * tends to cause slow streaming writers to write data to the
2315 * disk smoothly, at the dirtying rate, which is nice. But
2316 * that's undesirable in laptop mode, where we *want* lumpy
2317 * writeout. So in laptop mode, write out the whole world.
2318 */
KOSAKI Motohiro22fba332009-12-14 17:59:10 -08002319 writeback_threshold = sc->nr_to_reclaim + sc->nr_to_reclaim / 2;
2320 if (total_scanned > writeback_threshold) {
Curt Wohlgemuth0e175a12011-10-07 21:54:10 -06002321 wakeup_flusher_threads(laptop_mode ? 0 : total_scanned,
2322 WB_REASON_TRY_TO_FREE_PAGES);
Balbir Singh66e17072008-02-07 00:13:56 -08002323 sc->may_writepage = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002324 }
2325
2326 /* Take a nap, wait for some writeback to complete */
KOSAKI Motohiro7b517552009-12-14 17:59:12 -08002327 if (!sc->hibernation_mode && sc->nr_scanned &&
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002328 sc->priority < DEF_PRIORITY - 2) {
Mel Gorman0e093d992010-10-26 14:21:45 -07002329 struct zone *preferred_zone;
2330
2331 first_zones_zonelist(zonelist, gfp_zone(sc->gfp_mask),
David Rientjesf33261d2011-01-25 15:07:20 -08002332 &cpuset_current_mems_allowed,
2333 &preferred_zone);
Mel Gorman0e093d992010-10-26 14:21:45 -07002334 wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/10);
2335 }
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002336 } while (--sc->priority >= 0);
KOSAKI Motohirobb21c7c2010-06-04 14:15:05 -07002337
Linus Torvalds1da177e2005-04-16 15:20:36 -07002338out:
Keika Kobayashi873b4772008-07-25 01:48:52 -07002339 delayacct_freepages_end();
2340
KOSAKI Motohirobb21c7c2010-06-04 14:15:05 -07002341 if (sc->nr_reclaimed)
2342 return sc->nr_reclaimed;
2343
KOSAKI Motohiro929bea72011-04-14 15:22:12 -07002344 /*
2345 * As hibernation is going on, kswapd is freezed so that it can't mark
2346 * the zone into all_unreclaimable. Thus bypassing all_unreclaimable
2347 * check.
2348 */
2349 if (oom_killer_disabled)
2350 return 0;
2351
Mel Gorman0cee34f2012-01-12 17:19:49 -08002352 /* Aborted reclaim to try compaction? don't OOM, then */
2353 if (aborted_reclaim)
Mel Gorman73350842012-01-12 17:19:33 -08002354 return 1;
2355
KOSAKI Motohirobb21c7c2010-06-04 14:15:05 -07002356 /* top priority shrink_zones still had more to do? don't OOM, then */
Johannes Weiner89b5fae2012-01-12 17:17:50 -08002357 if (global_reclaim(sc) && !all_unreclaimable(zonelist, sc))
KOSAKI Motohirobb21c7c2010-06-04 14:15:05 -07002358 return 1;
2359
2360 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002361}
2362
Mel Gorman55150612012-07-31 16:44:35 -07002363static bool pfmemalloc_watermark_ok(pg_data_t *pgdat)
2364{
2365 struct zone *zone;
2366 unsigned long pfmemalloc_reserve = 0;
2367 unsigned long free_pages = 0;
2368 int i;
2369 bool wmark_ok;
2370
2371 for (i = 0; i <= ZONE_NORMAL; i++) {
2372 zone = &pgdat->node_zones[i];
2373 pfmemalloc_reserve += min_wmark_pages(zone);
2374 free_pages += zone_page_state(zone, NR_FREE_PAGES);
2375 }
2376
2377 wmark_ok = free_pages > pfmemalloc_reserve / 2;
2378
2379 /* kswapd must be awake if processes are being throttled */
2380 if (!wmark_ok && waitqueue_active(&pgdat->kswapd_wait)) {
2381 pgdat->classzone_idx = min(pgdat->classzone_idx,
2382 (enum zone_type)ZONE_NORMAL);
2383 wake_up_interruptible(&pgdat->kswapd_wait);
2384 }
2385
2386 return wmark_ok;
2387}
2388
2389/*
2390 * Throttle direct reclaimers if backing storage is backed by the network
2391 * and the PFMEMALLOC reserve for the preferred node is getting dangerously
2392 * depleted. kswapd will continue to make progress and wake the processes
Mel Gorman50694c22012-11-26 16:29:48 -08002393 * when the low watermark is reached.
2394 *
2395 * Returns true if a fatal signal was delivered during throttling. If this
2396 * happens, the page allocator should not consider triggering the OOM killer.
Mel Gorman55150612012-07-31 16:44:35 -07002397 */
Mel Gorman50694c22012-11-26 16:29:48 -08002398static bool throttle_direct_reclaim(gfp_t gfp_mask, struct zonelist *zonelist,
Mel Gorman55150612012-07-31 16:44:35 -07002399 nodemask_t *nodemask)
2400{
2401 struct zone *zone;
2402 int high_zoneidx = gfp_zone(gfp_mask);
2403 pg_data_t *pgdat;
2404
2405 /*
2406 * Kernel threads should not be throttled as they may be indirectly
2407 * responsible for cleaning pages necessary for reclaim to make forward
2408 * progress. kjournald for example may enter direct reclaim while
2409 * committing a transaction where throttling it could forcing other
2410 * processes to block on log_wait_commit().
2411 */
2412 if (current->flags & PF_KTHREAD)
Mel Gorman50694c22012-11-26 16:29:48 -08002413 goto out;
2414
2415 /*
2416 * If a fatal signal is pending, this process should not throttle.
2417 * It should return quickly so it can exit and free its memory
2418 */
2419 if (fatal_signal_pending(current))
2420 goto out;
Mel Gorman55150612012-07-31 16:44:35 -07002421
2422 /* Check if the pfmemalloc reserves are ok */
2423 first_zones_zonelist(zonelist, high_zoneidx, NULL, &zone);
2424 pgdat = zone->zone_pgdat;
2425 if (pfmemalloc_watermark_ok(pgdat))
Mel Gorman50694c22012-11-26 16:29:48 -08002426 goto out;
Mel Gorman55150612012-07-31 16:44:35 -07002427
Mel Gorman68243e72012-07-31 16:44:39 -07002428 /* Account for the throttling */
2429 count_vm_event(PGSCAN_DIRECT_THROTTLE);
2430
Mel Gorman55150612012-07-31 16:44:35 -07002431 /*
2432 * If the caller cannot enter the filesystem, it's possible that it
2433 * is due to the caller holding an FS lock or performing a journal
2434 * transaction in the case of a filesystem like ext[3|4]. In this case,
2435 * it is not safe to block on pfmemalloc_wait as kswapd could be
2436 * blocked waiting on the same lock. Instead, throttle for up to a
2437 * second before continuing.
2438 */
2439 if (!(gfp_mask & __GFP_FS)) {
2440 wait_event_interruptible_timeout(pgdat->pfmemalloc_wait,
2441 pfmemalloc_watermark_ok(pgdat), HZ);
Mel Gorman50694c22012-11-26 16:29:48 -08002442
2443 goto check_pending;
Mel Gorman55150612012-07-31 16:44:35 -07002444 }
2445
2446 /* Throttle until kswapd wakes the process */
2447 wait_event_killable(zone->zone_pgdat->pfmemalloc_wait,
2448 pfmemalloc_watermark_ok(pgdat));
Mel Gorman50694c22012-11-26 16:29:48 -08002449
2450check_pending:
2451 if (fatal_signal_pending(current))
2452 return true;
2453
2454out:
2455 return false;
Mel Gorman55150612012-07-31 16:44:35 -07002456}
2457
Mel Gormandac1d272008-04-28 02:12:12 -07002458unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
KAMEZAWA Hiroyuki327c0e92009-03-31 15:23:31 -07002459 gfp_t gfp_mask, nodemask_t *nodemask)
Balbir Singh66e17072008-02-07 00:13:56 -08002460{
Mel Gorman33906bc2010-08-09 17:19:16 -07002461 unsigned long nr_reclaimed;
Balbir Singh66e17072008-02-07 00:13:56 -08002462 struct scan_control sc = {
Ming Lei21caf2f2013-02-22 16:34:08 -08002463 .gfp_mask = (gfp_mask = memalloc_noio_flags(gfp_mask)),
Balbir Singh66e17072008-02-07 00:13:56 -08002464 .may_writepage = !laptop_mode,
KOSAKI Motohiro22fba332009-12-14 17:59:10 -08002465 .nr_to_reclaim = SWAP_CLUSTER_MAX,
Johannes Weinera6dc60f82009-03-31 15:19:30 -07002466 .may_unmap = 1,
KOSAKI Motohiro2e2e4252009-04-21 12:24:57 -07002467 .may_swap = 1,
Balbir Singh66e17072008-02-07 00:13:56 -08002468 .order = order,
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002469 .priority = DEF_PRIORITY,
Johannes Weinerf16015f2012-01-12 17:17:52 -08002470 .target_mem_cgroup = NULL,
KAMEZAWA Hiroyuki327c0e92009-03-31 15:23:31 -07002471 .nodemask = nodemask,
Balbir Singh66e17072008-02-07 00:13:56 -08002472 };
Ying Hana09ed5e2011-05-24 17:12:26 -07002473 struct shrink_control shrink = {
2474 .gfp_mask = sc.gfp_mask,
2475 };
Balbir Singh66e17072008-02-07 00:13:56 -08002476
Mel Gorman55150612012-07-31 16:44:35 -07002477 /*
Mel Gorman50694c22012-11-26 16:29:48 -08002478 * Do not enter reclaim if fatal signal was delivered while throttled.
2479 * 1 is returned so that the page allocator does not OOM kill at this
2480 * point.
Mel Gorman55150612012-07-31 16:44:35 -07002481 */
Mel Gorman50694c22012-11-26 16:29:48 -08002482 if (throttle_direct_reclaim(gfp_mask, zonelist, nodemask))
Mel Gorman55150612012-07-31 16:44:35 -07002483 return 1;
2484
Mel Gorman33906bc2010-08-09 17:19:16 -07002485 trace_mm_vmscan_direct_reclaim_begin(order,
2486 sc.may_writepage,
2487 gfp_mask);
2488
Ying Hana09ed5e2011-05-24 17:12:26 -07002489 nr_reclaimed = do_try_to_free_pages(zonelist, &sc, &shrink);
Mel Gorman33906bc2010-08-09 17:19:16 -07002490
2491 trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
2492
2493 return nr_reclaimed;
Balbir Singh66e17072008-02-07 00:13:56 -08002494}
2495
Andrew Mortonc255a452012-07-31 16:43:02 -07002496#ifdef CONFIG_MEMCG
Balbir Singh66e17072008-02-07 00:13:56 -08002497
Johannes Weiner72835c82012-01-12 17:18:32 -08002498unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *memcg,
Balbir Singh4e416952009-09-23 15:56:39 -07002499 gfp_t gfp_mask, bool noswap,
Ying Han0ae5e892011-05-26 16:25:25 -07002500 struct zone *zone,
2501 unsigned long *nr_scanned)
Balbir Singh4e416952009-09-23 15:56:39 -07002502{
2503 struct scan_control sc = {
Ying Han0ae5e892011-05-26 16:25:25 -07002504 .nr_scanned = 0,
KOSAKI Motohirob8f5c562010-08-10 18:03:02 -07002505 .nr_to_reclaim = SWAP_CLUSTER_MAX,
Balbir Singh4e416952009-09-23 15:56:39 -07002506 .may_writepage = !laptop_mode,
2507 .may_unmap = 1,
2508 .may_swap = !noswap,
Balbir Singh4e416952009-09-23 15:56:39 -07002509 .order = 0,
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002510 .priority = 0,
Johannes Weiner72835c82012-01-12 17:18:32 -08002511 .target_mem_cgroup = memcg,
Balbir Singh4e416952009-09-23 15:56:39 -07002512 };
Konstantin Khlebnikovf9be23d2012-05-29 15:07:02 -07002513 struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);
Ying Han0ae5e892011-05-26 16:25:25 -07002514
Balbir Singh4e416952009-09-23 15:56:39 -07002515 sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
2516 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
KOSAKI Motohirobdce6d92010-08-09 17:19:56 -07002517
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002518 trace_mm_vmscan_memcg_softlimit_reclaim_begin(sc.order,
KOSAKI Motohirobdce6d92010-08-09 17:19:56 -07002519 sc.may_writepage,
2520 sc.gfp_mask);
2521
Balbir Singh4e416952009-09-23 15:56:39 -07002522 /*
2523 * NOTE: Although we can get the priority field, using it
2524 * here is not a good idea, since it limits the pages we can scan.
2525 * if we don't reclaim here, the shrink_zone from balance_pgdat
2526 * will pick up pages from other mem cgroup's as well. We hack
2527 * the priority and make it zero.
2528 */
Konstantin Khlebnikovf9be23d2012-05-29 15:07:02 -07002529 shrink_lruvec(lruvec, &sc);
KOSAKI Motohirobdce6d92010-08-09 17:19:56 -07002530
2531 trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
2532
Ying Han0ae5e892011-05-26 16:25:25 -07002533 *nr_scanned = sc.nr_scanned;
Balbir Singh4e416952009-09-23 15:56:39 -07002534 return sc.nr_reclaimed;
2535}
2536
Johannes Weiner72835c82012-01-12 17:18:32 -08002537unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08002538 gfp_t gfp_mask,
Johannes Weiner185efc02011-09-14 16:21:58 -07002539 bool noswap)
Balbir Singh66e17072008-02-07 00:13:56 -08002540{
Balbir Singh4e416952009-09-23 15:56:39 -07002541 struct zonelist *zonelist;
KOSAKI Motohirobdce6d92010-08-09 17:19:56 -07002542 unsigned long nr_reclaimed;
Ying Han889976d2011-05-26 16:25:33 -07002543 int nid;
Balbir Singh66e17072008-02-07 00:13:56 -08002544 struct scan_control sc = {
Balbir Singh66e17072008-02-07 00:13:56 -08002545 .may_writepage = !laptop_mode,
Johannes Weinera6dc60f82009-03-31 15:19:30 -07002546 .may_unmap = 1,
KOSAKI Motohiro2e2e4252009-04-21 12:24:57 -07002547 .may_swap = !noswap,
KOSAKI Motohiro22fba332009-12-14 17:59:10 -08002548 .nr_to_reclaim = SWAP_CLUSTER_MAX,
Balbir Singh66e17072008-02-07 00:13:56 -08002549 .order = 0,
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002550 .priority = DEF_PRIORITY,
Johannes Weiner72835c82012-01-12 17:18:32 -08002551 .target_mem_cgroup = memcg,
KAMEZAWA Hiroyuki327c0e92009-03-31 15:23:31 -07002552 .nodemask = NULL, /* we don't care the placement */
Ying Hana09ed5e2011-05-24 17:12:26 -07002553 .gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
2554 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK),
2555 };
2556 struct shrink_control shrink = {
2557 .gfp_mask = sc.gfp_mask,
Balbir Singh66e17072008-02-07 00:13:56 -08002558 };
Balbir Singh66e17072008-02-07 00:13:56 -08002559
Ying Han889976d2011-05-26 16:25:33 -07002560 /*
2561 * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't
2562 * take care of from where we get pages. So the node where we start the
2563 * scan does not need to be the current node.
2564 */
Johannes Weiner72835c82012-01-12 17:18:32 -08002565 nid = mem_cgroup_select_victim_node(memcg);
Ying Han889976d2011-05-26 16:25:33 -07002566
2567 zonelist = NODE_DATA(nid)->node_zonelists;
KOSAKI Motohirobdce6d92010-08-09 17:19:56 -07002568
2569 trace_mm_vmscan_memcg_reclaim_begin(0,
2570 sc.may_writepage,
2571 sc.gfp_mask);
2572
Ying Hana09ed5e2011-05-24 17:12:26 -07002573 nr_reclaimed = do_try_to_free_pages(zonelist, &sc, &shrink);
KOSAKI Motohirobdce6d92010-08-09 17:19:56 -07002574
2575 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
2576
2577 return nr_reclaimed;
Balbir Singh66e17072008-02-07 00:13:56 -08002578}
2579#endif
2580
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002581static void age_active_anon(struct zone *zone, struct scan_control *sc)
Johannes Weinerf16015f2012-01-12 17:17:52 -08002582{
Johannes Weinerb95a2f22012-01-12 17:18:06 -08002583 struct mem_cgroup *memcg;
Johannes Weinerf16015f2012-01-12 17:17:52 -08002584
Johannes Weinerb95a2f22012-01-12 17:18:06 -08002585 if (!total_swap_pages)
2586 return;
2587
2588 memcg = mem_cgroup_iter(NULL, NULL, NULL);
2589 do {
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07002590 struct lruvec *lruvec = mem_cgroup_zone_lruvec(zone, memcg);
Johannes Weinerb95a2f22012-01-12 17:18:06 -08002591
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07002592 if (inactive_anon_is_low(lruvec))
Konstantin Khlebnikov1a93be02012-05-29 15:07:01 -07002593 shrink_active_list(SWAP_CLUSTER_MAX, lruvec,
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002594 sc, LRU_ACTIVE_ANON);
Johannes Weinerb95a2f22012-01-12 17:18:06 -08002595
2596 memcg = mem_cgroup_iter(NULL, memcg, NULL);
2597 } while (memcg);
Johannes Weinerf16015f2012-01-12 17:17:52 -08002598}
2599
Johannes Weiner60cefed2012-11-29 13:54:23 -08002600static bool zone_balanced(struct zone *zone, int order,
2601 unsigned long balance_gap, int classzone_idx)
2602{
2603 if (!zone_watermark_ok_safe(zone, order, high_wmark_pages(zone) +
2604 balance_gap, classzone_idx, 0))
2605 return false;
2606
Kirill A. Shutemovd84da3f2012-12-11 16:00:31 -08002607 if (IS_ENABLED(CONFIG_COMPACTION) && order &&
2608 !compaction_suitable(zone, order))
Johannes Weiner60cefed2012-11-29 13:54:23 -08002609 return false;
2610
2611 return true;
2612}
2613
Mel Gorman1741c872011-01-13 15:46:21 -08002614/*
Zlatko Calusic4ae0a482012-12-23 15:12:54 +01002615 * pgdat_balanced() is used when checking if a node is balanced.
2616 *
2617 * For order-0, all zones must be balanced!
2618 *
2619 * For high-order allocations only zones that meet watermarks and are in a
2620 * zone allowed by the callers classzone_idx are added to balanced_pages. The
2621 * total of balanced pages must be at least 25% of the zones allowed by
2622 * classzone_idx for the node to be considered balanced. Forcing all zones to
2623 * be balanced for high orders can cause excessive reclaim when there are
2624 * imbalanced zones.
Mel Gorman1741c872011-01-13 15:46:21 -08002625 * The choice of 25% is due to
2626 * o a 16M DMA zone that is balanced will not balance a zone on any
2627 * reasonable sized machine
2628 * o On all other machines, the top zone must be at least a reasonable
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002629 * percentage of the middle zones. For example, on 32-bit x86, highmem
Mel Gorman1741c872011-01-13 15:46:21 -08002630 * would need to be at least 256M for it to be balance a whole node.
2631 * Similarly, on x86-64 the Normal zone would need to be at least 1G
2632 * to balance a node on its own. These seemed like reasonable ratios.
2633 */
Zlatko Calusic4ae0a482012-12-23 15:12:54 +01002634static bool pgdat_balanced(pg_data_t *pgdat, int order, int classzone_idx)
Mel Gorman1741c872011-01-13 15:46:21 -08002635{
Jiang Liub40da042013-02-22 16:33:52 -08002636 unsigned long managed_pages = 0;
Zlatko Calusic4ae0a482012-12-23 15:12:54 +01002637 unsigned long balanced_pages = 0;
Mel Gorman1741c872011-01-13 15:46:21 -08002638 int i;
2639
Zlatko Calusic4ae0a482012-12-23 15:12:54 +01002640 /* Check the watermark levels */
2641 for (i = 0; i <= classzone_idx; i++) {
2642 struct zone *zone = pgdat->node_zones + i;
Mel Gorman1741c872011-01-13 15:46:21 -08002643
Zlatko Calusic4ae0a482012-12-23 15:12:54 +01002644 if (!populated_zone(zone))
2645 continue;
2646
Jiang Liub40da042013-02-22 16:33:52 -08002647 managed_pages += zone->managed_pages;
Zlatko Calusic4ae0a482012-12-23 15:12:54 +01002648
2649 /*
2650 * A special case here:
2651 *
2652 * balance_pgdat() skips over all_unreclaimable after
2653 * DEF_PRIORITY. Effectively, it considers them balanced so
2654 * they must be considered balanced here as well!
2655 */
2656 if (zone->all_unreclaimable) {
Jiang Liub40da042013-02-22 16:33:52 -08002657 balanced_pages += zone->managed_pages;
Zlatko Calusic4ae0a482012-12-23 15:12:54 +01002658 continue;
2659 }
2660
2661 if (zone_balanced(zone, order, 0, i))
Jiang Liub40da042013-02-22 16:33:52 -08002662 balanced_pages += zone->managed_pages;
Zlatko Calusic4ae0a482012-12-23 15:12:54 +01002663 else if (!order)
2664 return false;
2665 }
2666
2667 if (order)
Jiang Liub40da042013-02-22 16:33:52 -08002668 return balanced_pages >= (managed_pages >> 2);
Zlatko Calusic4ae0a482012-12-23 15:12:54 +01002669 else
2670 return true;
Mel Gorman1741c872011-01-13 15:46:21 -08002671}
2672
Mel Gorman55150612012-07-31 16:44:35 -07002673/*
2674 * Prepare kswapd for sleeping. This verifies that there are no processes
2675 * waiting in throttle_direct_reclaim() and that watermarks have been met.
2676 *
2677 * Returns true if kswapd is ready to sleep
2678 */
2679static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order, long remaining,
Mel Gormandc83edd2011-01-13 15:46:26 -08002680 int classzone_idx)
Mel Gormanf50de2d2009-12-14 17:58:53 -08002681{
Mel Gormanf50de2d2009-12-14 17:58:53 -08002682 /* If a direct reclaimer woke kswapd within HZ/10, it's premature */
2683 if (remaining)
Mel Gorman55150612012-07-31 16:44:35 -07002684 return false;
2685
2686 /*
2687 * There is a potential race between when kswapd checks its watermarks
2688 * and a process gets throttled. There is also a potential race if
2689 * processes get throttled, kswapd wakes, a large process exits therby
2690 * balancing the zones that causes kswapd to miss a wakeup. If kswapd
2691 * is going to sleep, no process should be sleeping on pfmemalloc_wait
2692 * so wake them now if necessary. If necessary, processes will wake
2693 * kswapd and get throttled again
2694 */
2695 if (waitqueue_active(&pgdat->pfmemalloc_wait)) {
2696 wake_up(&pgdat->pfmemalloc_wait);
2697 return false;
2698 }
Mel Gormanf50de2d2009-12-14 17:58:53 -08002699
Zlatko Calusic4ae0a482012-12-23 15:12:54 +01002700 return pgdat_balanced(pgdat, order, classzone_idx);
Mel Gormanf50de2d2009-12-14 17:58:53 -08002701}
2702
Linus Torvalds1da177e2005-04-16 15:20:36 -07002703/*
Mel Gorman75485362013-07-03 15:01:42 -07002704 * kswapd shrinks the zone by the number of pages required to reach
2705 * the high watermark.
Mel Gormanb8e83b92013-07-03 15:01:45 -07002706 *
2707 * Returns true if kswapd scanned at least the requested number of pages to
Mel Gorman283aba92013-07-03 15:01:51 -07002708 * reclaim or if the lack of progress was due to pages under writeback.
2709 * This is used to determine if the scanning priority needs to be raised.
Mel Gorman75485362013-07-03 15:01:42 -07002710 */
Mel Gormanb8e83b92013-07-03 15:01:45 -07002711static bool kswapd_shrink_zone(struct zone *zone,
Mel Gorman75485362013-07-03 15:01:42 -07002712 struct scan_control *sc,
Mel Gorman2ab44f42013-07-03 15:01:47 -07002713 unsigned long lru_pages,
2714 unsigned long *nr_attempted)
Mel Gorman75485362013-07-03 15:01:42 -07002715{
2716 unsigned long nr_slab;
2717 struct reclaim_state *reclaim_state = current->reclaim_state;
2718 struct shrink_control shrink = {
2719 .gfp_mask = sc->gfp_mask,
2720 };
2721
2722 /* Reclaim above the high watermark. */
2723 sc->nr_to_reclaim = max(SWAP_CLUSTER_MAX, high_wmark_pages(zone));
2724 shrink_zone(zone, sc);
2725
2726 reclaim_state->reclaimed_slab = 0;
2727 nr_slab = shrink_slab(&shrink, sc->nr_scanned, lru_pages);
2728 sc->nr_reclaimed += reclaim_state->reclaimed_slab;
2729
Mel Gorman2ab44f42013-07-03 15:01:47 -07002730 /* Account for the number of pages attempted to reclaim */
2731 *nr_attempted += sc->nr_to_reclaim;
2732
Mel Gorman75485362013-07-03 15:01:42 -07002733 if (nr_slab == 0 && !zone_reclaimable(zone))
2734 zone->all_unreclaimable = 1;
Mel Gormanb8e83b92013-07-03 15:01:45 -07002735
Mel Gorman283aba92013-07-03 15:01:51 -07002736 zone_clear_flag(zone, ZONE_WRITEBACK);
2737
Mel Gormanb8e83b92013-07-03 15:01:45 -07002738 return sc->nr_scanned >= sc->nr_to_reclaim;
Mel Gorman75485362013-07-03 15:01:42 -07002739}
2740
2741/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002742 * For kswapd, balance_pgdat() will work across all this node's zones until
Mel Gorman41858962009-06-16 15:32:12 -07002743 * they are all at high_wmark_pages(zone).
Linus Torvalds1da177e2005-04-16 15:20:36 -07002744 *
Mel Gorman0abdee22011-01-13 15:46:22 -08002745 * Returns the final order kswapd was reclaiming at
Linus Torvalds1da177e2005-04-16 15:20:36 -07002746 *
2747 * There is special handling here for zones which are full of pinned pages.
2748 * This can happen if the pages are all mlocked, or if they are all used by
2749 * device drivers (say, ZONE_DMA). Or if they are all in use by hugetlb.
2750 * What we do is to detect the case where all pages in the zone have been
2751 * scanned twice and there has been zero successful reclaim. Mark the zone as
2752 * dead and from now on, only perform a short scan. Basically we're polling
2753 * the zone for when the problem goes away.
2754 *
2755 * kswapd scans the zones in the highmem->normal->dma direction. It skips
Mel Gorman41858962009-06-16 15:32:12 -07002756 * zones which have free_pages > high_wmark_pages(zone), but once a zone is
2757 * found to have free_pages <= high_wmark_pages(zone), we scan that zone and the
2758 * lower zones regardless of the number of free pages in the lower zones. This
2759 * interoperates with the page allocator fallback scheme to ensure that aging
2760 * of pages is balanced across the zones.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002761 */
Mel Gorman99504742011-01-13 15:46:20 -08002762static unsigned long balance_pgdat(pg_data_t *pgdat, int order,
Mel Gormandc83edd2011-01-13 15:46:26 -08002763 int *classzone_idx)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002764{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002765 int i;
Mel Gorman99504742011-01-13 15:46:20 -08002766 int end_zone = 0; /* Inclusive. 0 = ZONE_DMA */
Ying Han0ae5e892011-05-26 16:25:25 -07002767 unsigned long nr_soft_reclaimed;
2768 unsigned long nr_soft_scanned;
Andrew Morton179e9632006-03-22 00:08:18 -08002769 struct scan_control sc = {
2770 .gfp_mask = GFP_KERNEL,
Mel Gormanb8e83b92013-07-03 15:01:45 -07002771 .priority = DEF_PRIORITY,
Johannes Weinera6dc60f82009-03-31 15:19:30 -07002772 .may_unmap = 1,
KOSAKI Motohiro2e2e4252009-04-21 12:24:57 -07002773 .may_swap = 1,
Mel Gormanb8e83b92013-07-03 15:01:45 -07002774 .may_writepage = !laptop_mode,
Andy Whitcroft5ad333e2007-07-17 04:03:16 -07002775 .order = order,
Johannes Weinerf16015f2012-01-12 17:17:52 -08002776 .target_mem_cgroup = NULL,
Andrew Morton179e9632006-03-22 00:08:18 -08002777 };
Christoph Lameterf8891e52006-06-30 01:55:45 -07002778 count_vm_event(PAGEOUTRUN);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002779
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002780 do {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002781 unsigned long lru_pages = 0;
Mel Gorman2ab44f42013-07-03 15:01:47 -07002782 unsigned long nr_attempted = 0;
Mel Gormanb8e83b92013-07-03 15:01:45 -07002783 bool raise_priority = true;
Mel Gorman2ab44f42013-07-03 15:01:47 -07002784 bool pgdat_needs_compaction = (order > 0);
Mel Gormanb8e83b92013-07-03 15:01:45 -07002785
2786 sc.nr_reclaimed = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002787
Rafael J. Wysockid6277db2006-06-23 02:03:18 -07002788 /*
2789 * Scan in the highmem->dma direction for the highest
2790 * zone which needs scanning
2791 */
2792 for (i = pgdat->nr_zones - 1; i >= 0; i--) {
2793 struct zone *zone = pgdat->node_zones + i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002794
Rafael J. Wysockid6277db2006-06-23 02:03:18 -07002795 if (!populated_zone(zone))
2796 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002797
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002798 if (zone->all_unreclaimable &&
2799 sc.priority != DEF_PRIORITY)
Rafael J. Wysockid6277db2006-06-23 02:03:18 -07002800 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002801
Rik van Riel556adec2008-10-18 20:26:34 -07002802 /*
2803 * Do some background aging of the anon list, to give
2804 * pages a chance to be referenced before reclaiming.
2805 */
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002806 age_active_anon(zone, &sc);
Rik van Riel556adec2008-10-18 20:26:34 -07002807
Mel Gormancc715d92012-03-21 16:34:00 -07002808 /*
2809 * If the number of buffer_heads in the machine
2810 * exceeds the maximum allowed level and this node
2811 * has a highmem zone, force kswapd to reclaim from
2812 * it to relieve lowmem pressure.
2813 */
2814 if (buffer_heads_over_limit && is_highmem_idx(i)) {
2815 end_zone = i;
2816 break;
2817 }
2818
Johannes Weiner60cefed2012-11-29 13:54:23 -08002819 if (!zone_balanced(zone, order, 0, 0)) {
Rafael J. Wysockid6277db2006-06-23 02:03:18 -07002820 end_zone = i;
Andrew Mortone1dbeda2006-12-06 20:32:01 -08002821 break;
Shaohua Li439423f2011-08-25 15:59:12 -07002822 } else {
Mel Gormand43006d2013-07-03 15:01:50 -07002823 /*
2824 * If balanced, clear the dirty and congested
2825 * flags
2826 */
Shaohua Li439423f2011-08-25 15:59:12 -07002827 zone_clear_flag(zone, ZONE_CONGESTED);
Mel Gormand43006d2013-07-03 15:01:50 -07002828 zone_clear_flag(zone, ZONE_TAIL_LRU_DIRTY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002829 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002830 }
Zlatko Calusicdafcb732013-02-22 16:32:34 -08002831
Mel Gormanb8e83b92013-07-03 15:01:45 -07002832 if (i < 0)
Andrew Mortone1dbeda2006-12-06 20:32:01 -08002833 goto out;
2834
Linus Torvalds1da177e2005-04-16 15:20:36 -07002835 for (i = 0; i <= end_zone; i++) {
2836 struct zone *zone = pgdat->node_zones + i;
2837
Mel Gorman2ab44f42013-07-03 15:01:47 -07002838 if (!populated_zone(zone))
2839 continue;
2840
Wu Fengguangadea02a2009-09-21 17:01:42 -07002841 lru_pages += zone_reclaimable_pages(zone);
Mel Gorman2ab44f42013-07-03 15:01:47 -07002842
2843 /*
2844 * If any zone is currently balanced then kswapd will
2845 * not call compaction as it is expected that the
2846 * necessary pages are already available.
2847 */
2848 if (pgdat_needs_compaction &&
2849 zone_watermark_ok(zone, order,
2850 low_wmark_pages(zone),
2851 *classzone_idx, 0))
2852 pgdat_needs_compaction = false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002853 }
2854
2855 /*
Mel Gormanb7ea3c42013-07-03 15:01:53 -07002856 * If we're getting trouble reclaiming, start doing writepage
2857 * even in laptop mode.
2858 */
2859 if (sc.priority < DEF_PRIORITY - 2)
2860 sc.may_writepage = 1;
2861
2862 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -07002863 * Now scan the zone in the dma->highmem direction, stopping
2864 * at the last zone which needs scanning.
2865 *
2866 * We do this because the page allocator works in the opposite
2867 * direction. This prevents the page allocator from allocating
2868 * pages behind kswapd's direction of progress, which would
2869 * cause too much scanning of the lower zones.
2870 */
2871 for (i = 0; i <= end_zone; i++) {
2872 struct zone *zone = pgdat->node_zones + i;
Mel Gorman75485362013-07-03 15:01:42 -07002873 int testorder;
Mel Gorman8afdcec2011-03-22 16:33:04 -07002874 unsigned long balance_gap;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002875
Con Kolivasf3fe6512006-01-06 00:11:15 -08002876 if (!populated_zone(zone))
Linus Torvalds1da177e2005-04-16 15:20:36 -07002877 continue;
2878
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07002879 if (zone->all_unreclaimable &&
2880 sc.priority != DEF_PRIORITY)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002881 continue;
2882
Linus Torvalds1da177e2005-04-16 15:20:36 -07002883 sc.nr_scanned = 0;
Balbir Singh4e416952009-09-23 15:56:39 -07002884
Ying Han0ae5e892011-05-26 16:25:25 -07002885 nr_soft_scanned = 0;
Balbir Singh4e416952009-09-23 15:56:39 -07002886 /*
2887 * Call soft limit reclaim before calling shrink_zone.
Balbir Singh4e416952009-09-23 15:56:39 -07002888 */
Ying Han0ae5e892011-05-26 16:25:25 -07002889 nr_soft_reclaimed = mem_cgroup_soft_limit_reclaim(zone,
2890 order, sc.gfp_mask,
2891 &nr_soft_scanned);
2892 sc.nr_reclaimed += nr_soft_reclaimed;
KOSAKI Motohiro00918b62010-08-10 18:03:05 -07002893
Rik van Riel32a43302007-10-16 01:24:50 -07002894 /*
Mel Gorman8afdcec2011-03-22 16:33:04 -07002895 * We put equal pressure on every zone, unless
2896 * one zone has way too many pages free
2897 * already. The "too many pages" is defined
2898 * as the high wmark plus a "gap" where the
2899 * gap is either the low watermark or 1%
2900 * of the zone, whichever is smaller.
Rik van Riel32a43302007-10-16 01:24:50 -07002901 */
Mel Gorman8afdcec2011-03-22 16:33:04 -07002902 balance_gap = min(low_wmark_pages(zone),
Jiang Liub40da042013-02-22 16:33:52 -08002903 (zone->managed_pages +
Mel Gorman8afdcec2011-03-22 16:33:04 -07002904 KSWAPD_ZONE_BALANCE_GAP_RATIO-1) /
2905 KSWAPD_ZONE_BALANCE_GAP_RATIO);
Rik van Rielfe2c2a12012-03-21 16:33:51 -07002906 /*
2907 * Kswapd reclaims only single pages with compaction
2908 * enabled. Trying too hard to reclaim until contiguous
2909 * free pages have become available can hurt performance
2910 * by evicting too much useful data from memory.
2911 * Do not reclaim more than needed for compaction.
2912 */
2913 testorder = order;
Kirill A. Shutemovd84da3f2012-12-11 16:00:31 -08002914 if (IS_ENABLED(CONFIG_COMPACTION) && order &&
Rik van Rielfe2c2a12012-03-21 16:33:51 -07002915 compaction_suitable(zone, order) !=
2916 COMPACT_SKIPPED)
2917 testorder = 0;
2918
Mel Gormancc715d92012-03-21 16:34:00 -07002919 if ((buffer_heads_over_limit && is_highmem_idx(i)) ||
Johannes Weiner60cefed2012-11-29 13:54:23 -08002920 !zone_balanced(zone, testorder,
Mel Gormanb8e83b92013-07-03 15:01:45 -07002921 balance_gap, end_zone)) {
2922 /*
2923 * There should be no need to raise the
2924 * scanning priority if enough pages are
2925 * already being scanned that high
2926 * watermark would be met at 100% efficiency.
2927 */
Mel Gorman2ab44f42013-07-03 15:01:47 -07002928 if (kswapd_shrink_zone(zone, &sc, lru_pages,
2929 &nr_attempted))
Mel Gormanb8e83b92013-07-03 15:01:45 -07002930 raise_priority = false;
2931 }
Mel Gormand7868da2011-07-08 15:39:38 -07002932
Mel Gorman215ddd62011-07-08 15:39:40 -07002933 if (zone->all_unreclaimable) {
2934 if (end_zone && end_zone == i)
2935 end_zone--;
Mel Gormand7868da2011-07-08 15:39:38 -07002936 continue;
Mel Gorman215ddd62011-07-08 15:39:40 -07002937 }
Mel Gormand7868da2011-07-08 15:39:38 -07002938
Zlatko Calusic258401a2013-02-22 16:34:06 -08002939 if (zone_balanced(zone, testorder, 0, end_zone))
Mel Gorman0e093d992010-10-26 14:21:45 -07002940 /*
2941 * If a zone reaches its high watermark,
2942 * consider it to be no longer congested. It's
2943 * possible there are dirty pages backed by
2944 * congested BDIs but as pressure is relieved,
Wanpeng Liab8704b2012-06-17 09:27:18 +08002945 * speculatively avoid congestion waits
Mel Gormand43006d2013-07-03 15:01:50 -07002946 * or writing pages from kswapd context.
Mel Gorman0e093d992010-10-26 14:21:45 -07002947 */
2948 zone_clear_flag(zone, ZONE_CONGESTED);
Mel Gormand43006d2013-07-03 15:01:50 -07002949 zone_clear_flag(zone, ZONE_TAIL_LRU_DIRTY);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002950 }
Mel Gorman55150612012-07-31 16:44:35 -07002951
2952 /*
2953 * If the low watermark is met there is no need for processes
2954 * to be throttled on pfmemalloc_wait as they should not be
2955 * able to safely make forward progress. Wake them
2956 */
2957 if (waitqueue_active(&pgdat->pfmemalloc_wait) &&
2958 pfmemalloc_watermark_ok(pgdat))
2959 wake_up(&pgdat->pfmemalloc_wait);
2960
Linus Torvalds1da177e2005-04-16 15:20:36 -07002961 /*
Mel Gormanb8e83b92013-07-03 15:01:45 -07002962 * Fragmentation may mean that the system cannot be rebalanced
2963 * for high-order allocations in all zones. If twice the
2964 * allocation size has been reclaimed and the zones are still
2965 * not balanced then recheck the watermarks at order-0 to
2966 * prevent kswapd reclaiming excessively. Assume that a
2967 * process requested a high-order can direct reclaim/compact.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002968 */
Mel Gormanb8e83b92013-07-03 15:01:45 -07002969 if (order && sc.nr_reclaimed >= 2UL << order)
KOSAKI Motohiro73ce02e2009-01-06 14:40:33 -08002970 order = sc.order = 0;
2971
Mel Gormanb8e83b92013-07-03 15:01:45 -07002972 /* Check if kswapd should be suspending */
2973 if (try_to_freeze() || kthread_should_stop())
2974 break;
2975
2976 /*
Mel Gorman2ab44f42013-07-03 15:01:47 -07002977 * Compact if necessary and kswapd is reclaiming at least the
2978 * high watermark number of pages as requsted
2979 */
2980 if (pgdat_needs_compaction && sc.nr_reclaimed > nr_attempted)
2981 compact_pgdat(pgdat, order);
2982
2983 /*
Mel Gormanb8e83b92013-07-03 15:01:45 -07002984 * Raise priority if scanning rate is too low or there was no
2985 * progress in reclaiming pages
2986 */
2987 if (raise_priority || !sc.nr_reclaimed)
2988 sc.priority--;
Mel Gorman9aa41342013-07-03 15:01:48 -07002989 } while (sc.priority >= 1 &&
Mel Gormanb8e83b92013-07-03 15:01:45 -07002990 !pgdat_balanced(pgdat, order, *classzone_idx));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002991
Mel Gormanb8e83b92013-07-03 15:01:45 -07002992out:
Mel Gorman0abdee22011-01-13 15:46:22 -08002993 /*
Mel Gorman55150612012-07-31 16:44:35 -07002994 * Return the order we were reclaiming at so prepare_kswapd_sleep()
Mel Gorman0abdee22011-01-13 15:46:22 -08002995 * makes a decision on the order we were last reclaiming at. However,
2996 * if another caller entered the allocator slow path while kswapd
2997 * was awake, order will remain at the higher level
2998 */
Mel Gormandc83edd2011-01-13 15:46:26 -08002999 *classzone_idx = end_zone;
Mel Gorman0abdee22011-01-13 15:46:22 -08003000 return order;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003001}
3002
Mel Gormandc83edd2011-01-13 15:46:26 -08003003static void kswapd_try_to_sleep(pg_data_t *pgdat, int order, int classzone_idx)
KOSAKI Motohirof0bc0a62011-01-13 15:45:50 -08003004{
3005 long remaining = 0;
3006 DEFINE_WAIT(wait);
3007
3008 if (freezing(current) || kthread_should_stop())
3009 return;
3010
3011 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
3012
3013 /* Try to sleep for a short interval */
Mel Gorman55150612012-07-31 16:44:35 -07003014 if (prepare_kswapd_sleep(pgdat, order, remaining, classzone_idx)) {
KOSAKI Motohirof0bc0a62011-01-13 15:45:50 -08003015 remaining = schedule_timeout(HZ/10);
3016 finish_wait(&pgdat->kswapd_wait, &wait);
3017 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
3018 }
3019
3020 /*
3021 * After a short sleep, check if it was a premature sleep. If not, then
3022 * go fully to sleep until explicitly woken up.
3023 */
Mel Gorman55150612012-07-31 16:44:35 -07003024 if (prepare_kswapd_sleep(pgdat, order, remaining, classzone_idx)) {
KOSAKI Motohirof0bc0a62011-01-13 15:45:50 -08003025 trace_mm_vmscan_kswapd_sleep(pgdat->node_id);
3026
3027 /*
3028 * vmstat counters are not perfectly accurate and the estimated
3029 * value for counters such as NR_FREE_PAGES can deviate from the
3030 * true value by nr_online_cpus * threshold. To avoid the zone
3031 * watermarks being breached while under pressure, we reduce the
3032 * per-cpu vmstat threshold while kswapd is awake and restore
3033 * them before going back to sleep.
3034 */
3035 set_pgdat_percpu_threshold(pgdat, calculate_normal_threshold);
Aaditya Kumar1c7e7f62012-07-17 15:48:07 -07003036
Mel Gorman62997022012-10-08 16:32:47 -07003037 /*
3038 * Compaction records what page blocks it recently failed to
3039 * isolate pages from and skips them in the future scanning.
3040 * When kswapd is going to sleep, it is reasonable to assume
3041 * that pages and compaction may succeed so reset the cache.
3042 */
3043 reset_isolation_suitable(pgdat);
3044
Aaditya Kumar1c7e7f62012-07-17 15:48:07 -07003045 if (!kthread_should_stop())
3046 schedule();
3047
KOSAKI Motohirof0bc0a62011-01-13 15:45:50 -08003048 set_pgdat_percpu_threshold(pgdat, calculate_pressure_threshold);
3049 } else {
3050 if (remaining)
3051 count_vm_event(KSWAPD_LOW_WMARK_HIT_QUICKLY);
3052 else
3053 count_vm_event(KSWAPD_HIGH_WMARK_HIT_QUICKLY);
3054 }
3055 finish_wait(&pgdat->kswapd_wait, &wait);
3056}
3057
Linus Torvalds1da177e2005-04-16 15:20:36 -07003058/*
3059 * The background pageout daemon, started as a kernel thread
Rik van Riel4f98a2f2008-10-18 20:26:32 -07003060 * from the init process.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003061 *
3062 * This basically trickles out pages so that we have _some_
3063 * free memory available even if there is no other activity
3064 * that frees anything up. This is needed for things like routing
3065 * etc, where we otherwise might have all activity going on in
3066 * asynchronous contexts that cannot page things out.
3067 *
3068 * If there are applications that are active memory-allocators
3069 * (most normal use), this basically shouldn't matter.
3070 */
3071static int kswapd(void *p)
3072{
Mel Gorman215ddd62011-07-08 15:39:40 -07003073 unsigned long order, new_order;
Alex,Shid2ebd0f62011-10-31 17:08:39 -07003074 unsigned balanced_order;
Mel Gorman215ddd62011-07-08 15:39:40 -07003075 int classzone_idx, new_classzone_idx;
Alex,Shid2ebd0f62011-10-31 17:08:39 -07003076 int balanced_classzone_idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003077 pg_data_t *pgdat = (pg_data_t*)p;
3078 struct task_struct *tsk = current;
KOSAKI Motohirof0bc0a62011-01-13 15:45:50 -08003079
Linus Torvalds1da177e2005-04-16 15:20:36 -07003080 struct reclaim_state reclaim_state = {
3081 .reclaimed_slab = 0,
3082 };
Rusty Russella70f7302009-03-13 14:49:46 +10303083 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003084
Nick Piggincf40bd12009-01-21 08:12:39 +01003085 lockdep_set_current_reclaim_state(GFP_KERNEL);
3086
Rusty Russell174596a2009-01-01 10:12:29 +10303087 if (!cpumask_empty(cpumask))
Mike Travisc5f59f02008-04-04 18:11:10 -07003088 set_cpus_allowed_ptr(tsk, cpumask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003089 current->reclaim_state = &reclaim_state;
3090
3091 /*
3092 * Tell the memory management that we're a "memory allocator",
3093 * and that if we need more memory we should get access to it
3094 * regardless (see "__alloc_pages()"). "kswapd" should
3095 * never get caught in the normal page freeing logic.
3096 *
3097 * (Kswapd normally doesn't need memory anyway, but sometimes
3098 * you need a small amount of memory in order to be able to
3099 * page out something else, and this flag essentially protects
3100 * us from recursively trying to free more memory as we're
3101 * trying to free the first piece of memory in the first place).
3102 */
Christoph Lameter930d9152006-01-08 01:00:47 -08003103 tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
Rafael J. Wysocki83144182007-07-17 04:03:35 -07003104 set_freezable();
Linus Torvalds1da177e2005-04-16 15:20:36 -07003105
Mel Gorman215ddd62011-07-08 15:39:40 -07003106 order = new_order = 0;
Alex,Shid2ebd0f62011-10-31 17:08:39 -07003107 balanced_order = 0;
Mel Gorman215ddd62011-07-08 15:39:40 -07003108 classzone_idx = new_classzone_idx = pgdat->nr_zones - 1;
Alex,Shid2ebd0f62011-10-31 17:08:39 -07003109 balanced_classzone_idx = classzone_idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003110 for ( ; ; ) {
Jeff Liu6f6313d2012-12-11 16:02:48 -08003111 bool ret;
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07003112
Mel Gorman215ddd62011-07-08 15:39:40 -07003113 /*
3114 * If the last balance_pgdat was unsuccessful it's unlikely a
3115 * new request of a similar or harder type will succeed soon
3116 * so consider going to sleep on the basis we reclaimed at
3117 */
Alex,Shid2ebd0f62011-10-31 17:08:39 -07003118 if (balanced_classzone_idx >= new_classzone_idx &&
3119 balanced_order == new_order) {
Mel Gorman215ddd62011-07-08 15:39:40 -07003120 new_order = pgdat->kswapd_max_order;
3121 new_classzone_idx = pgdat->classzone_idx;
3122 pgdat->kswapd_max_order = 0;
3123 pgdat->classzone_idx = pgdat->nr_zones - 1;
3124 }
3125
Mel Gorman99504742011-01-13 15:46:20 -08003126 if (order < new_order || classzone_idx > new_classzone_idx) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003127 /*
3128 * Don't sleep if someone wants a larger 'order'
Mel Gorman99504742011-01-13 15:46:20 -08003129 * allocation or has tigher zone constraints
Linus Torvalds1da177e2005-04-16 15:20:36 -07003130 */
3131 order = new_order;
Mel Gorman99504742011-01-13 15:46:20 -08003132 classzone_idx = new_classzone_idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003133 } else {
Alex,Shid2ebd0f62011-10-31 17:08:39 -07003134 kswapd_try_to_sleep(pgdat, balanced_order,
3135 balanced_classzone_idx);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003136 order = pgdat->kswapd_max_order;
Mel Gorman99504742011-01-13 15:46:20 -08003137 classzone_idx = pgdat->classzone_idx;
Alex,Shif0dfcde2011-10-31 17:08:45 -07003138 new_order = order;
3139 new_classzone_idx = classzone_idx;
Mel Gorman4d405022011-01-13 15:46:23 -08003140 pgdat->kswapd_max_order = 0;
Mel Gorman215ddd62011-07-08 15:39:40 -07003141 pgdat->classzone_idx = pgdat->nr_zones - 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003142 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003143
David Rientjes8fe23e02009-12-14 17:58:33 -08003144 ret = try_to_freeze();
3145 if (kthread_should_stop())
3146 break;
3147
3148 /*
3149 * We can speed up thawing tasks if we don't call balance_pgdat
3150 * after returning from the refrigerator
3151 */
Mel Gorman33906bc2010-08-09 17:19:16 -07003152 if (!ret) {
3153 trace_mm_vmscan_kswapd_wake(pgdat->node_id, order);
Alex,Shid2ebd0f62011-10-31 17:08:39 -07003154 balanced_classzone_idx = classzone_idx;
3155 balanced_order = balance_pgdat(pgdat, order,
3156 &balanced_classzone_idx);
Mel Gorman33906bc2010-08-09 17:19:16 -07003157 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003158 }
Takamori Yamaguchib0a8cc52012-11-08 15:53:39 -08003159
3160 current->reclaim_state = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003161 return 0;
3162}
3163
3164/*
3165 * A zone is low on free memory, so wake its kswapd task to service it.
3166 */
Mel Gorman99504742011-01-13 15:46:20 -08003167void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003168{
3169 pg_data_t *pgdat;
3170
Con Kolivasf3fe6512006-01-06 00:11:15 -08003171 if (!populated_zone(zone))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003172 return;
3173
Paul Jackson02a0e532006-12-13 00:34:25 -08003174 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003175 return;
Mel Gorman88f5acf2011-01-13 15:45:41 -08003176 pgdat = zone->zone_pgdat;
Mel Gorman99504742011-01-13 15:46:20 -08003177 if (pgdat->kswapd_max_order < order) {
Mel Gorman88f5acf2011-01-13 15:45:41 -08003178 pgdat->kswapd_max_order = order;
Mel Gorman99504742011-01-13 15:46:20 -08003179 pgdat->classzone_idx = min(pgdat->classzone_idx, classzone_idx);
3180 }
Con Kolivas8d0986e2005-09-13 01:25:07 -07003181 if (!waitqueue_active(&pgdat->kswapd_wait))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003182 return;
Mel Gorman88f5acf2011-01-13 15:45:41 -08003183 if (zone_watermark_ok_safe(zone, order, low_wmark_pages(zone), 0, 0))
3184 return;
3185
3186 trace_mm_vmscan_wakeup_kswapd(pgdat->node_id, zone_idx(zone), order);
Con Kolivas8d0986e2005-09-13 01:25:07 -07003187 wake_up_interruptible(&pgdat->kswapd_wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003188}
3189
Wu Fengguangadea02a2009-09-21 17:01:42 -07003190/*
3191 * The reclaimable count would be mostly accurate.
3192 * The less reclaimable pages may be
3193 * - mlocked pages, which will be moved to unevictable list when encountered
3194 * - mapped pages, which may require several travels to be reclaimed
3195 * - dirty pages, which is not "instantly" reclaimable
3196 */
3197unsigned long global_reclaimable_pages(void)
Rik van Riel4f98a2f2008-10-18 20:26:32 -07003198{
Wu Fengguangadea02a2009-09-21 17:01:42 -07003199 int nr;
3200
3201 nr = global_page_state(NR_ACTIVE_FILE) +
3202 global_page_state(NR_INACTIVE_FILE);
3203
Shaohua Liec8acf22013-02-22 16:34:38 -08003204 if (get_nr_swap_pages() > 0)
Wu Fengguangadea02a2009-09-21 17:01:42 -07003205 nr += global_page_state(NR_ACTIVE_ANON) +
3206 global_page_state(NR_INACTIVE_ANON);
3207
3208 return nr;
3209}
3210
3211unsigned long zone_reclaimable_pages(struct zone *zone)
3212{
3213 int nr;
3214
3215 nr = zone_page_state(zone, NR_ACTIVE_FILE) +
3216 zone_page_state(zone, NR_INACTIVE_FILE);
3217
Shaohua Liec8acf22013-02-22 16:34:38 -08003218 if (get_nr_swap_pages() > 0)
Wu Fengguangadea02a2009-09-21 17:01:42 -07003219 nr += zone_page_state(zone, NR_ACTIVE_ANON) +
3220 zone_page_state(zone, NR_INACTIVE_ANON);
3221
3222 return nr;
Rik van Riel4f98a2f2008-10-18 20:26:32 -07003223}
3224
Rafael J. Wysockic6f37f12009-05-24 22:16:31 +02003225#ifdef CONFIG_HIBERNATION
Linus Torvalds1da177e2005-04-16 15:20:36 -07003226/*
KOSAKI Motohiro7b517552009-12-14 17:59:12 -08003227 * Try to free `nr_to_reclaim' of memory, system-wide, and return the number of
Rafael J. Wysockid6277db2006-06-23 02:03:18 -07003228 * freed pages.
3229 *
3230 * Rather than trying to age LRUs the aim is to preserve the overall
3231 * LRU order by reclaiming preferentially
3232 * inactive > active > active referenced > active mapped
Linus Torvalds1da177e2005-04-16 15:20:36 -07003233 */
KOSAKI Motohiro7b517552009-12-14 17:59:12 -08003234unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003235{
Rafael J. Wysockid6277db2006-06-23 02:03:18 -07003236 struct reclaim_state reclaim_state;
Rafael J. Wysockid6277db2006-06-23 02:03:18 -07003237 struct scan_control sc = {
KOSAKI Motohiro7b517552009-12-14 17:59:12 -08003238 .gfp_mask = GFP_HIGHUSER_MOVABLE,
3239 .may_swap = 1,
3240 .may_unmap = 1,
Rafael J. Wysockid6277db2006-06-23 02:03:18 -07003241 .may_writepage = 1,
KOSAKI Motohiro7b517552009-12-14 17:59:12 -08003242 .nr_to_reclaim = nr_to_reclaim,
3243 .hibernation_mode = 1,
KOSAKI Motohiro7b517552009-12-14 17:59:12 -08003244 .order = 0,
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07003245 .priority = DEF_PRIORITY,
Linus Torvalds1da177e2005-04-16 15:20:36 -07003246 };
Ying Hana09ed5e2011-05-24 17:12:26 -07003247 struct shrink_control shrink = {
3248 .gfp_mask = sc.gfp_mask,
3249 };
3250 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
KOSAKI Motohiro7b517552009-12-14 17:59:12 -08003251 struct task_struct *p = current;
3252 unsigned long nr_reclaimed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003253
KOSAKI Motohiro7b517552009-12-14 17:59:12 -08003254 p->flags |= PF_MEMALLOC;
3255 lockdep_set_current_reclaim_state(sc.gfp_mask);
3256 reclaim_state.reclaimed_slab = 0;
3257 p->reclaim_state = &reclaim_state;
Andrew Morton69e05942006-03-22 00:08:19 -08003258
Ying Hana09ed5e2011-05-24 17:12:26 -07003259 nr_reclaimed = do_try_to_free_pages(zonelist, &sc, &shrink);
Rafael J. Wysockid6277db2006-06-23 02:03:18 -07003260
KOSAKI Motohiro7b517552009-12-14 17:59:12 -08003261 p->reclaim_state = NULL;
3262 lockdep_clear_current_reclaim_state();
3263 p->flags &= ~PF_MEMALLOC;
Rafael J. Wysockid6277db2006-06-23 02:03:18 -07003264
KOSAKI Motohiro7b517552009-12-14 17:59:12 -08003265 return nr_reclaimed;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003266}
Rafael J. Wysockic6f37f12009-05-24 22:16:31 +02003267#endif /* CONFIG_HIBERNATION */
Linus Torvalds1da177e2005-04-16 15:20:36 -07003268
Linus Torvalds1da177e2005-04-16 15:20:36 -07003269/* It's optimal to keep kswapds on the same CPUs as their memory, but
3270 not required for correctness. So if the last cpu in a node goes
3271 away, we get changed to run anywhere: as the first one comes back,
3272 restore their cpu bindings. */
Greg Kroah-Hartmanfcb35a92012-12-21 15:01:06 -08003273static int cpu_callback(struct notifier_block *nfb, unsigned long action,
3274 void *hcpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003275{
Yasunori Goto58c0a4a2007-10-16 01:25:40 -07003276 int nid;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003277
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07003278 if (action == CPU_ONLINE || action == CPU_ONLINE_FROZEN) {
Lai Jiangshan48fb2e22012-12-12 13:51:43 -08003279 for_each_node_state(nid, N_MEMORY) {
Mike Travisc5f59f02008-04-04 18:11:10 -07003280 pg_data_t *pgdat = NODE_DATA(nid);
Rusty Russella70f7302009-03-13 14:49:46 +10303281 const struct cpumask *mask;
3282
3283 mask = cpumask_of_node(pgdat->node_id);
Mike Travisc5f59f02008-04-04 18:11:10 -07003284
Rusty Russell3e597942009-01-01 10:12:24 +10303285 if (cpumask_any_and(cpu_online_mask, mask) < nr_cpu_ids)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003286 /* One of our CPUs online: restore mask */
Mike Travisc5f59f02008-04-04 18:11:10 -07003287 set_cpus_allowed_ptr(pgdat->kswapd, mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003288 }
3289 }
3290 return NOTIFY_OK;
3291}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003292
Yasunori Goto3218ae12006-06-27 02:53:33 -07003293/*
3294 * This kswapd start function will be called by init and node-hot-add.
3295 * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
3296 */
3297int kswapd_run(int nid)
3298{
3299 pg_data_t *pgdat = NODE_DATA(nid);
3300 int ret = 0;
3301
3302 if (pgdat->kswapd)
3303 return 0;
3304
3305 pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
3306 if (IS_ERR(pgdat->kswapd)) {
3307 /* failure at boot is fatal */
3308 BUG_ON(system_state == SYSTEM_BOOTING);
Gavin Shand5dc0ad2012-10-08 16:29:27 -07003309 pr_err("Failed to start kswapd on node %d\n", nid);
3310 ret = PTR_ERR(pgdat->kswapd);
Xishi Qiud72515b2013-04-17 15:58:34 -07003311 pgdat->kswapd = NULL;
Yasunori Goto3218ae12006-06-27 02:53:33 -07003312 }
3313 return ret;
3314}
3315
David Rientjes8fe23e02009-12-14 17:58:33 -08003316/*
Jiang Liud8adde12012-07-11 14:01:52 -07003317 * Called by memory hotplug when all memory in a node is offlined. Caller must
3318 * hold lock_memory_hotplug().
David Rientjes8fe23e02009-12-14 17:58:33 -08003319 */
3320void kswapd_stop(int nid)
3321{
3322 struct task_struct *kswapd = NODE_DATA(nid)->kswapd;
3323
Jiang Liud8adde12012-07-11 14:01:52 -07003324 if (kswapd) {
David Rientjes8fe23e02009-12-14 17:58:33 -08003325 kthread_stop(kswapd);
Jiang Liud8adde12012-07-11 14:01:52 -07003326 NODE_DATA(nid)->kswapd = NULL;
3327 }
David Rientjes8fe23e02009-12-14 17:58:33 -08003328}
3329
Linus Torvalds1da177e2005-04-16 15:20:36 -07003330static int __init kswapd_init(void)
3331{
Yasunori Goto3218ae12006-06-27 02:53:33 -07003332 int nid;
Andrew Morton69e05942006-03-22 00:08:19 -08003333
Linus Torvalds1da177e2005-04-16 15:20:36 -07003334 swap_setup();
Lai Jiangshan48fb2e22012-12-12 13:51:43 -08003335 for_each_node_state(nid, N_MEMORY)
Yasunori Goto3218ae12006-06-27 02:53:33 -07003336 kswapd_run(nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003337 hotcpu_notifier(cpu_callback, 0);
3338 return 0;
3339}
3340
3341module_init(kswapd_init)
Christoph Lameter9eeff232006-01-18 17:42:31 -08003342
3343#ifdef CONFIG_NUMA
3344/*
3345 * Zone reclaim mode
3346 *
3347 * If non-zero call zone_reclaim when the number of free pages falls below
3348 * the watermarks.
Christoph Lameter9eeff232006-01-18 17:42:31 -08003349 */
3350int zone_reclaim_mode __read_mostly;
3351
Christoph Lameter1b2ffb72006-02-01 03:05:34 -08003352#define RECLAIM_OFF 0
Fernando Luis Vazquez Cao7d034312008-07-29 22:33:41 -07003353#define RECLAIM_ZONE (1<<0) /* Run shrink_inactive_list on the zone */
Christoph Lameter1b2ffb72006-02-01 03:05:34 -08003354#define RECLAIM_WRITE (1<<1) /* Writeout pages during reclaim */
3355#define RECLAIM_SWAP (1<<2) /* Swap pages out during reclaim */
3356
Christoph Lameter9eeff232006-01-18 17:42:31 -08003357/*
Christoph Lametera92f7122006-02-01 03:05:32 -08003358 * Priority for ZONE_RECLAIM. This determines the fraction of pages
3359 * of a node considered for each zone_reclaim. 4 scans 1/16th of
3360 * a zone.
3361 */
3362#define ZONE_RECLAIM_PRIORITY 4
3363
Christoph Lameter9eeff232006-01-18 17:42:31 -08003364/*
Christoph Lameter96146342006-07-03 00:24:13 -07003365 * Percentage of pages in a zone that must be unmapped for zone_reclaim to
3366 * occur.
3367 */
3368int sysctl_min_unmapped_ratio = 1;
3369
3370/*
Christoph Lameter0ff38492006-09-25 23:31:52 -07003371 * If the number of slab pages in a zone grows beyond this percentage then
3372 * slab reclaim needs to occur.
3373 */
3374int sysctl_min_slab_ratio = 5;
3375
Mel Gorman90afa5d2009-06-16 15:33:20 -07003376static inline unsigned long zone_unmapped_file_pages(struct zone *zone)
3377{
3378 unsigned long file_mapped = zone_page_state(zone, NR_FILE_MAPPED);
3379 unsigned long file_lru = zone_page_state(zone, NR_INACTIVE_FILE) +
3380 zone_page_state(zone, NR_ACTIVE_FILE);
3381
3382 /*
3383 * It's possible for there to be more file mapped pages than
3384 * accounted for by the pages on the file LRU lists because
3385 * tmpfs pages accounted for as ANON can also be FILE_MAPPED
3386 */
3387 return (file_lru > file_mapped) ? (file_lru - file_mapped) : 0;
3388}
3389
3390/* Work out how many page cache pages we can reclaim in this reclaim_mode */
3391static long zone_pagecache_reclaimable(struct zone *zone)
3392{
3393 long nr_pagecache_reclaimable;
3394 long delta = 0;
3395
3396 /*
3397 * If RECLAIM_SWAP is set, then all file pages are considered
3398 * potentially reclaimable. Otherwise, we have to worry about
3399 * pages like swapcache and zone_unmapped_file_pages() provides
3400 * a better estimate
3401 */
3402 if (zone_reclaim_mode & RECLAIM_SWAP)
3403 nr_pagecache_reclaimable = zone_page_state(zone, NR_FILE_PAGES);
3404 else
3405 nr_pagecache_reclaimable = zone_unmapped_file_pages(zone);
3406
3407 /* If we can't clean pages, remove dirty pages from consideration */
3408 if (!(zone_reclaim_mode & RECLAIM_WRITE))
3409 delta += zone_page_state(zone, NR_FILE_DIRTY);
3410
3411 /* Watch for any possible underflows due to delta */
3412 if (unlikely(delta > nr_pagecache_reclaimable))
3413 delta = nr_pagecache_reclaimable;
3414
3415 return nr_pagecache_reclaimable - delta;
3416}
3417
Christoph Lameter0ff38492006-09-25 23:31:52 -07003418/*
Christoph Lameter9eeff232006-01-18 17:42:31 -08003419 * Try to free up some pages from this zone through reclaim.
3420 */
Andrew Morton179e9632006-03-22 00:08:18 -08003421static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
Christoph Lameter9eeff232006-01-18 17:42:31 -08003422{
Christoph Lameter7fb2d462006-03-22 00:08:22 -08003423 /* Minimum pages needed in order to stay on node */
Andrew Morton69e05942006-03-22 00:08:19 -08003424 const unsigned long nr_pages = 1 << order;
Christoph Lameter9eeff232006-01-18 17:42:31 -08003425 struct task_struct *p = current;
3426 struct reclaim_state reclaim_state;
Andrew Morton179e9632006-03-22 00:08:18 -08003427 struct scan_control sc = {
3428 .may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE),
Johannes Weinera6dc60f82009-03-31 15:19:30 -07003429 .may_unmap = !!(zone_reclaim_mode & RECLAIM_SWAP),
KOSAKI Motohiro2e2e4252009-04-21 12:24:57 -07003430 .may_swap = 1,
Andrew Morton62b726c2013-02-22 16:32:24 -08003431 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
Ming Lei21caf2f2013-02-22 16:34:08 -08003432 .gfp_mask = (gfp_mask = memalloc_noio_flags(gfp_mask)),
Johannes Weinerbd2f6192009-03-31 15:19:38 -07003433 .order = order,
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07003434 .priority = ZONE_RECLAIM_PRIORITY,
Andrew Morton179e9632006-03-22 00:08:18 -08003435 };
Ying Hana09ed5e2011-05-24 17:12:26 -07003436 struct shrink_control shrink = {
3437 .gfp_mask = sc.gfp_mask,
3438 };
KOSAKI Motohiro15748042010-08-09 17:19:50 -07003439 unsigned long nr_slab_pages0, nr_slab_pages1;
Christoph Lameter9eeff232006-01-18 17:42:31 -08003440
Christoph Lameter9eeff232006-01-18 17:42:31 -08003441 cond_resched();
Christoph Lameterd4f77962006-02-24 13:04:22 -08003442 /*
3443 * We need to be able to allocate from the reserves for RECLAIM_SWAP
3444 * and we also need to be able to write out pages for RECLAIM_WRITE
3445 * and RECLAIM_SWAP.
3446 */
3447 p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
KOSAKI Motohiro76ca5422010-03-05 13:41:47 -08003448 lockdep_set_current_reclaim_state(gfp_mask);
Christoph Lameter9eeff232006-01-18 17:42:31 -08003449 reclaim_state.reclaimed_slab = 0;
3450 p->reclaim_state = &reclaim_state;
Christoph Lameterc84db232006-02-01 03:05:29 -08003451
Mel Gorman90afa5d2009-06-16 15:33:20 -07003452 if (zone_pagecache_reclaimable(zone) > zone->min_unmapped_pages) {
Christoph Lameter0ff38492006-09-25 23:31:52 -07003453 /*
3454 * Free memory by calling shrink zone with increasing
3455 * priorities until we have enough memory freed.
3456 */
Christoph Lameter0ff38492006-09-25 23:31:52 -07003457 do {
Konstantin Khlebnikov9e3b2f82012-05-29 15:06:57 -07003458 shrink_zone(zone, &sc);
3459 } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
Christoph Lameter0ff38492006-09-25 23:31:52 -07003460 }
Christoph Lameterc84db232006-02-01 03:05:29 -08003461
KOSAKI Motohiro15748042010-08-09 17:19:50 -07003462 nr_slab_pages0 = zone_page_state(zone, NR_SLAB_RECLAIMABLE);
3463 if (nr_slab_pages0 > zone->min_slab_pages) {
Christoph Lameter2a16e3f2006-02-01 03:05:35 -08003464 /*
Christoph Lameter7fb2d462006-03-22 00:08:22 -08003465 * shrink_slab() does not currently allow us to determine how
Christoph Lameter0ff38492006-09-25 23:31:52 -07003466 * many pages were freed in this zone. So we take the current
3467 * number of slab pages and shake the slab until it is reduced
3468 * by the same nr_pages that we used for reclaiming unmapped
3469 * pages.
Christoph Lameter2a16e3f2006-02-01 03:05:35 -08003470 *
Christoph Lameter0ff38492006-09-25 23:31:52 -07003471 * Note that shrink_slab will free memory on all zones and may
3472 * take a long time.
Christoph Lameter2a16e3f2006-02-01 03:05:35 -08003473 */
KOSAKI Motohiro4dc4b3d2010-08-09 17:19:54 -07003474 for (;;) {
3475 unsigned long lru_pages = zone_reclaimable_pages(zone);
3476
3477 /* No reclaimable slab or very low memory pressure */
Ying Han1495f232011-05-24 17:12:27 -07003478 if (!shrink_slab(&shrink, sc.nr_scanned, lru_pages))
KOSAKI Motohiro4dc4b3d2010-08-09 17:19:54 -07003479 break;
3480
3481 /* Freed enough memory */
3482 nr_slab_pages1 = zone_page_state(zone,
3483 NR_SLAB_RECLAIMABLE);
3484 if (nr_slab_pages1 + nr_pages <= nr_slab_pages0)
3485 break;
3486 }
Christoph Lameter83e33a42006-09-25 23:31:53 -07003487
3488 /*
3489 * Update nr_reclaimed by the number of slab pages we
3490 * reclaimed from this zone.
3491 */
KOSAKI Motohiro15748042010-08-09 17:19:50 -07003492 nr_slab_pages1 = zone_page_state(zone, NR_SLAB_RECLAIMABLE);
3493 if (nr_slab_pages1 < nr_slab_pages0)
3494 sc.nr_reclaimed += nr_slab_pages0 - nr_slab_pages1;
Christoph Lameter2a16e3f2006-02-01 03:05:35 -08003495 }
3496
Christoph Lameter9eeff232006-01-18 17:42:31 -08003497 p->reclaim_state = NULL;
Christoph Lameterd4f77962006-02-24 13:04:22 -08003498 current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
KOSAKI Motohiro76ca5422010-03-05 13:41:47 -08003499 lockdep_clear_current_reclaim_state();
Rik van Riela79311c2009-01-06 14:40:01 -08003500 return sc.nr_reclaimed >= nr_pages;
Christoph Lameter9eeff232006-01-18 17:42:31 -08003501}
Andrew Morton179e9632006-03-22 00:08:18 -08003502
3503int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
3504{
Andrew Morton179e9632006-03-22 00:08:18 -08003505 int node_id;
David Rientjesd773ed62007-10-16 23:26:01 -07003506 int ret;
Andrew Morton179e9632006-03-22 00:08:18 -08003507
3508 /*
Christoph Lameter0ff38492006-09-25 23:31:52 -07003509 * Zone reclaim reclaims unmapped file backed pages and
3510 * slab pages if we are over the defined limits.
Christoph Lameter34aa1332006-06-30 01:55:37 -07003511 *
Christoph Lameter96146342006-07-03 00:24:13 -07003512 * A small portion of unmapped file backed pages is needed for
3513 * file I/O otherwise pages read by file I/O will be immediately
3514 * thrown out if the zone is overallocated. So we do not reclaim
3515 * if less than a specified percentage of the zone is used by
3516 * unmapped file backed pages.
Andrew Morton179e9632006-03-22 00:08:18 -08003517 */
Mel Gorman90afa5d2009-06-16 15:33:20 -07003518 if (zone_pagecache_reclaimable(zone) <= zone->min_unmapped_pages &&
3519 zone_page_state(zone, NR_SLAB_RECLAIMABLE) <= zone->min_slab_pages)
Mel Gormanfa5e0842009-06-16 15:33:22 -07003520 return ZONE_RECLAIM_FULL;
Andrew Morton179e9632006-03-22 00:08:18 -08003521
KOSAKI Motohiro93e4a892010-03-05 13:41:55 -08003522 if (zone->all_unreclaimable)
Mel Gormanfa5e0842009-06-16 15:33:22 -07003523 return ZONE_RECLAIM_FULL;
David Rientjesd773ed62007-10-16 23:26:01 -07003524
Andrew Morton179e9632006-03-22 00:08:18 -08003525 /*
David Rientjesd773ed62007-10-16 23:26:01 -07003526 * Do not scan if the allocation should not be delayed.
Andrew Morton179e9632006-03-22 00:08:18 -08003527 */
David Rientjesd773ed62007-10-16 23:26:01 -07003528 if (!(gfp_mask & __GFP_WAIT) || (current->flags & PF_MEMALLOC))
Mel Gormanfa5e0842009-06-16 15:33:22 -07003529 return ZONE_RECLAIM_NOSCAN;
Andrew Morton179e9632006-03-22 00:08:18 -08003530
3531 /*
3532 * Only run zone reclaim on the local zone or on zones that do not
3533 * have associated processors. This will favor the local processor
3534 * over remote processors and spread off node memory allocations
3535 * as wide as possible.
3536 */
Christoph Lameter89fa3022006-09-25 23:31:55 -07003537 node_id = zone_to_nid(zone);
Christoph Lameter37c07082007-10-16 01:25:36 -07003538 if (node_state(node_id, N_CPU) && node_id != numa_node_id())
Mel Gormanfa5e0842009-06-16 15:33:22 -07003539 return ZONE_RECLAIM_NOSCAN;
David Rientjesd773ed62007-10-16 23:26:01 -07003540
3541 if (zone_test_and_set_flag(zone, ZONE_RECLAIM_LOCKED))
Mel Gormanfa5e0842009-06-16 15:33:22 -07003542 return ZONE_RECLAIM_NOSCAN;
3543
David Rientjesd773ed62007-10-16 23:26:01 -07003544 ret = __zone_reclaim(zone, gfp_mask, order);
3545 zone_clear_flag(zone, ZONE_RECLAIM_LOCKED);
3546
Mel Gorman24cf725182009-06-16 15:33:23 -07003547 if (!ret)
3548 count_vm_event(PGSCAN_ZONE_RECLAIM_FAILED);
3549
David Rientjesd773ed62007-10-16 23:26:01 -07003550 return ret;
Andrew Morton179e9632006-03-22 00:08:18 -08003551}
Christoph Lameter9eeff232006-01-18 17:42:31 -08003552#endif
Lee Schermerhorn894bc312008-10-18 20:26:39 -07003553
Lee Schermerhorn894bc312008-10-18 20:26:39 -07003554/*
3555 * page_evictable - test whether a page is evictable
3556 * @page: the page to test
Lee Schermerhorn894bc312008-10-18 20:26:39 -07003557 *
3558 * Test whether page is evictable--i.e., should be placed on active/inactive
Hugh Dickins39b5f292012-10-08 16:33:18 -07003559 * lists vs unevictable list.
Lee Schermerhorn894bc312008-10-18 20:26:39 -07003560 *
3561 * Reasons page might not be evictable:
Lee Schermerhornba9ddf42008-10-18 20:26:42 -07003562 * (1) page's mapping marked unevictable
Nick Pigginb291f002008-10-18 20:26:44 -07003563 * (2) page is part of an mlocked VMA
Lee Schermerhornba9ddf42008-10-18 20:26:42 -07003564 *
Lee Schermerhorn894bc312008-10-18 20:26:39 -07003565 */
Hugh Dickins39b5f292012-10-08 16:33:18 -07003566int page_evictable(struct page *page)
Lee Schermerhorn894bc312008-10-18 20:26:39 -07003567{
Hugh Dickins39b5f292012-10-08 16:33:18 -07003568 return !mapping_unevictable(page_mapping(page)) && !PageMlocked(page);
Lee Schermerhorn894bc312008-10-18 20:26:39 -07003569}
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07003570
Hugh Dickins850465792012-01-20 14:34:19 -08003571#ifdef CONFIG_SHMEM
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07003572/**
Hugh Dickins24513262012-01-20 14:34:21 -08003573 * check_move_unevictable_pages - check pages for evictability and move to appropriate zone lru list
3574 * @pages: array of pages to check
3575 * @nr_pages: number of pages to check
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07003576 *
Hugh Dickins24513262012-01-20 14:34:21 -08003577 * Checks pages for evictability and moves them to the appropriate lru list.
Hugh Dickins850465792012-01-20 14:34:19 -08003578 *
3579 * This function is only used for SysV IPC SHM_UNLOCK.
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07003580 */
Hugh Dickins24513262012-01-20 14:34:21 -08003581void check_move_unevictable_pages(struct page **pages, int nr_pages)
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07003582{
Johannes Weiner925b7672012-01-12 17:18:15 -08003583 struct lruvec *lruvec;
Hugh Dickins24513262012-01-20 14:34:21 -08003584 struct zone *zone = NULL;
3585 int pgscanned = 0;
3586 int pgrescued = 0;
3587 int i;
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07003588
Hugh Dickins24513262012-01-20 14:34:21 -08003589 for (i = 0; i < nr_pages; i++) {
3590 struct page *page = pages[i];
3591 struct zone *pagezone;
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003592
Hugh Dickins24513262012-01-20 14:34:21 -08003593 pgscanned++;
3594 pagezone = page_zone(page);
3595 if (pagezone != zone) {
3596 if (zone)
3597 spin_unlock_irq(&zone->lru_lock);
3598 zone = pagezone;
3599 spin_lock_irq(&zone->lru_lock);
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07003600 }
Hugh Dickinsfa9add62012-05-29 15:07:09 -07003601 lruvec = mem_cgroup_page_lruvec(page, zone);
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07003602
Hugh Dickins24513262012-01-20 14:34:21 -08003603 if (!PageLRU(page) || !PageUnevictable(page))
3604 continue;
3605
Hugh Dickins39b5f292012-10-08 16:33:18 -07003606 if (page_evictable(page)) {
Hugh Dickins24513262012-01-20 14:34:21 -08003607 enum lru_list lru = page_lru_base_type(page);
3608
3609 VM_BUG_ON(PageActive(page));
3610 ClearPageUnevictable(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07003611 del_page_from_lru_list(page, lruvec, LRU_UNEVICTABLE);
3612 add_page_to_lru_list(page, lruvec, lru);
Hugh Dickins24513262012-01-20 14:34:21 -08003613 pgrescued++;
3614 }
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -07003615 }
Hugh Dickins24513262012-01-20 14:34:21 -08003616
3617 if (zone) {
3618 __count_vm_events(UNEVICTABLE_PGRESCUED, pgrescued);
3619 __count_vm_events(UNEVICTABLE_PGSCANNED, pgscanned);
3620 spin_unlock_irq(&zone->lru_lock);
3621 }
Hugh Dickins850465792012-01-20 14:34:19 -08003622}
3623#endif /* CONFIG_SHMEM */
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003624
Johannes Weiner264e56d2011-10-31 17:09:13 -07003625static void warn_scan_unevictable_pages(void)
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003626{
Johannes Weiner264e56d2011-10-31 17:09:13 -07003627 printk_once(KERN_WARNING
KOSAKI Motohiro25bd91b2012-01-10 15:07:40 -08003628 "%s: The scan_unevictable_pages sysctl/node-interface has been "
Johannes Weiner264e56d2011-10-31 17:09:13 -07003629 "disabled for lack of a legitimate use case. If you have "
KOSAKI Motohiro25bd91b2012-01-10 15:07:40 -08003630 "one, please send an email to linux-mm@kvack.org.\n",
3631 current->comm);
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003632}
3633
3634/*
3635 * scan_unevictable_pages [vm] sysctl handler. On demand re-scan of
3636 * all nodes' unevictable lists for evictable pages
3637 */
3638unsigned long scan_unevictable_pages;
3639
3640int scan_unevictable_handler(struct ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07003641 void __user *buffer,
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003642 size_t *length, loff_t *ppos)
3643{
Johannes Weiner264e56d2011-10-31 17:09:13 -07003644 warn_scan_unevictable_pages();
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07003645 proc_doulongvec_minmax(table, write, buffer, length, ppos);
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003646 scan_unevictable_pages = 0;
3647 return 0;
3648}
3649
Thadeu Lima de Souza Cascardoe4455ab2010-10-26 14:21:28 -07003650#ifdef CONFIG_NUMA
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003651/*
3652 * per node 'scan_unevictable_pages' attribute. On demand re-scan of
3653 * a specified node's per zone unevictable lists for evictable pages.
3654 */
3655
Kay Sievers10fbcf42011-12-21 14:48:43 -08003656static ssize_t read_scan_unevictable_node(struct device *dev,
3657 struct device_attribute *attr,
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003658 char *buf)
3659{
Johannes Weiner264e56d2011-10-31 17:09:13 -07003660 warn_scan_unevictable_pages();
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003661 return sprintf(buf, "0\n"); /* always zero; should fit... */
3662}
3663
Kay Sievers10fbcf42011-12-21 14:48:43 -08003664static ssize_t write_scan_unevictable_node(struct device *dev,
3665 struct device_attribute *attr,
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003666 const char *buf, size_t count)
3667{
Johannes Weiner264e56d2011-10-31 17:09:13 -07003668 warn_scan_unevictable_pages();
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003669 return 1;
3670}
3671
3672
Kay Sievers10fbcf42011-12-21 14:48:43 -08003673static DEVICE_ATTR(scan_unevictable_pages, S_IRUGO | S_IWUSR,
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003674 read_scan_unevictable_node,
3675 write_scan_unevictable_node);
3676
3677int scan_unevictable_register_node(struct node *node)
3678{
Kay Sievers10fbcf42011-12-21 14:48:43 -08003679 return device_create_file(&node->dev, &dev_attr_scan_unevictable_pages);
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003680}
3681
3682void scan_unevictable_unregister_node(struct node *node)
3683{
Kay Sievers10fbcf42011-12-21 14:48:43 -08003684 device_remove_file(&node->dev, &dev_attr_scan_unevictable_pages);
Lee Schermerhornaf936a12008-10-18 20:26:53 -07003685}
Thadeu Lima de Souza Cascardoe4455ab2010-10-26 14:21:28 -07003686#endif