blob: 261a56ee11b690b8e182fbbe84a0b42c097c446f [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/vmscan.c
3 *
4 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
5 *
6 * Swap reorganised 29.12.95, Stephen Tweedie.
7 * kswapd added: 7.1.96 sct
8 * Removed kswapd_ctl limits, and swap out as many pages as needed
9 * to bring the system back to freepages.high: 2.4.97, Rik van Riel.
10 * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
11 * Multiqueue VM started 5.8.00, Rik van Riel.
12 */
13
14#include <linux/mm.h>
15#include <linux/module.h>
16#include <linux/slab.h>
17#include <linux/kernel_stat.h>
18#include <linux/swap.h>
19#include <linux/pagemap.h>
20#include <linux/init.h>
21#include <linux/highmem.h>
22#include <linux/file.h>
23#include <linux/writeback.h>
24#include <linux/blkdev.h>
25#include <linux/buffer_head.h> /* for try_to_release_page(),
26 buffer_heads_over_limit */
27#include <linux/mm_inline.h>
28#include <linux/pagevec.h>
29#include <linux/backing-dev.h>
30#include <linux/rmap.h>
31#include <linux/topology.h>
32#include <linux/cpu.h>
33#include <linux/cpuset.h>
34#include <linux/notifier.h>
35#include <linux/rwsem.h>
36
37#include <asm/tlbflush.h>
38#include <asm/div64.h>
39
40#include <linux/swapops.h>
41
42/* possible outcome of pageout() */
43typedef enum {
44 /* failed to write page out, page is locked */
45 PAGE_KEEP,
46 /* move page to the active list, page is locked */
47 PAGE_ACTIVATE,
48 /* page has been sent to the disk successfully, page is unlocked */
49 PAGE_SUCCESS,
50 /* page is clean and locked */
51 PAGE_CLEAN,
52} pageout_t;
53
54struct scan_control {
55 /* Ask refill_inactive_zone, or shrink_cache to scan this many pages */
56 unsigned long nr_to_scan;
57
58 /* Incremented by the number of inactive pages that were scanned */
59 unsigned long nr_scanned;
60
61 /* Incremented by the number of pages reclaimed */
62 unsigned long nr_reclaimed;
63
64 unsigned long nr_mapped; /* From page_state */
65
Linus Torvalds1da177e2005-04-16 15:20:36 -070066 /* Ask shrink_caches, or shrink_zone to scan at this priority */
67 unsigned int priority;
68
69 /* This context's GFP mask */
Al Viro6daa0e22005-10-21 03:18:50 -040070 gfp_t gfp_mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -070071
72 int may_writepage;
73
74 /* This context's SWAP_CLUSTER_MAX. If freeing memory for
75 * suspend, we effectively ignore SWAP_CLUSTER_MAX.
76 * In this context, it doesn't matter that we scan the
77 * whole list at once. */
78 int swap_cluster_max;
79};
80
81/*
82 * The list of shrinker callbacks used by to apply pressure to
83 * ageable caches.
84 */
85struct shrinker {
86 shrinker_t shrinker;
87 struct list_head list;
88 int seeks; /* seeks to recreate an obj */
89 long nr; /* objs pending delete */
90};
91
92#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
93
94#ifdef ARCH_HAS_PREFETCH
95#define prefetch_prev_lru_page(_page, _base, _field) \
96 do { \
97 if ((_page)->lru.prev != _base) { \
98 struct page *prev; \
99 \
100 prev = lru_to_page(&(_page->lru)); \
101 prefetch(&prev->_field); \
102 } \
103 } while (0)
104#else
105#define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
106#endif
107
108#ifdef ARCH_HAS_PREFETCHW
109#define prefetchw_prev_lru_page(_page, _base, _field) \
110 do { \
111 if ((_page)->lru.prev != _base) { \
112 struct page *prev; \
113 \
114 prev = lru_to_page(&(_page->lru)); \
115 prefetchw(&prev->_field); \
116 } \
117 } while (0)
118#else
119#define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
120#endif
121
122/*
123 * From 0 .. 100. Higher means more swappy.
124 */
125int vm_swappiness = 60;
126static long total_memory;
127
128static LIST_HEAD(shrinker_list);
129static DECLARE_RWSEM(shrinker_rwsem);
130
131/*
132 * Add a shrinker callback to be called from the vm
133 */
134struct shrinker *set_shrinker(int seeks, shrinker_t theshrinker)
135{
136 struct shrinker *shrinker;
137
138 shrinker = kmalloc(sizeof(*shrinker), GFP_KERNEL);
139 if (shrinker) {
140 shrinker->shrinker = theshrinker;
141 shrinker->seeks = seeks;
142 shrinker->nr = 0;
143 down_write(&shrinker_rwsem);
144 list_add_tail(&shrinker->list, &shrinker_list);
145 up_write(&shrinker_rwsem);
146 }
147 return shrinker;
148}
149EXPORT_SYMBOL(set_shrinker);
150
151/*
152 * Remove one
153 */
154void remove_shrinker(struct shrinker *shrinker)
155{
156 down_write(&shrinker_rwsem);
157 list_del(&shrinker->list);
158 up_write(&shrinker_rwsem);
159 kfree(shrinker);
160}
161EXPORT_SYMBOL(remove_shrinker);
162
163#define SHRINK_BATCH 128
164/*
165 * Call the shrink functions to age shrinkable caches
166 *
167 * Here we assume it costs one seek to replace a lru page and that it also
168 * takes a seek to recreate a cache object. With this in mind we age equal
169 * percentages of the lru and ageable caches. This should balance the seeks
170 * generated by these structures.
171 *
172 * If the vm encounted mapped pages on the LRU it increase the pressure on
173 * slab to avoid swapping.
174 *
175 * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits.
176 *
177 * `lru_pages' represents the number of on-LRU pages in all the zones which
178 * are eligible for the caller's allocation attempt. It is used for balancing
179 * slab reclaim versus page reclaim.
akpm@osdl.orgb15e0902005-06-21 17:14:35 -0700180 *
181 * Returns the number of slab objects which we shrunk.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182 */
Andrew Morton9d0243b2006-01-08 01:00:39 -0800183int shrink_slab(unsigned long scanned, gfp_t gfp_mask, unsigned long lru_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700184{
185 struct shrinker *shrinker;
akpm@osdl.orgb15e0902005-06-21 17:14:35 -0700186 int ret = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700187
188 if (scanned == 0)
189 scanned = SWAP_CLUSTER_MAX;
190
191 if (!down_read_trylock(&shrinker_rwsem))
akpm@osdl.orgb15e0902005-06-21 17:14:35 -0700192 return 1; /* Assume we'll be able to shrink next time */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700193
194 list_for_each_entry(shrinker, &shrinker_list, list) {
195 unsigned long long delta;
196 unsigned long total_scan;
Andrea Arcangeliea164d72005-11-28 13:44:15 -0800197 unsigned long max_pass = (*shrinker->shrinker)(0, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700198
199 delta = (4 * scanned) / shrinker->seeks;
Andrea Arcangeliea164d72005-11-28 13:44:15 -0800200 delta *= max_pass;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201 do_div(delta, lru_pages + 1);
202 shrinker->nr += delta;
Andrea Arcangeliea164d72005-11-28 13:44:15 -0800203 if (shrinker->nr < 0) {
204 printk(KERN_ERR "%s: nr=%ld\n",
205 __FUNCTION__, shrinker->nr);
206 shrinker->nr = max_pass;
207 }
208
209 /*
210 * Avoid risking looping forever due to too large nr value:
211 * never try to free more than twice the estimate number of
212 * freeable entries.
213 */
214 if (shrinker->nr > max_pass * 2)
215 shrinker->nr = max_pass * 2;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700216
217 total_scan = shrinker->nr;
218 shrinker->nr = 0;
219
220 while (total_scan >= SHRINK_BATCH) {
221 long this_scan = SHRINK_BATCH;
222 int shrink_ret;
akpm@osdl.orgb15e0902005-06-21 17:14:35 -0700223 int nr_before;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700224
akpm@osdl.orgb15e0902005-06-21 17:14:35 -0700225 nr_before = (*shrinker->shrinker)(0, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226 shrink_ret = (*shrinker->shrinker)(this_scan, gfp_mask);
227 if (shrink_ret == -1)
228 break;
akpm@osdl.orgb15e0902005-06-21 17:14:35 -0700229 if (shrink_ret < nr_before)
230 ret += nr_before - shrink_ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700231 mod_page_state(slabs_scanned, this_scan);
232 total_scan -= this_scan;
233
234 cond_resched();
235 }
236
237 shrinker->nr += total_scan;
238 }
239 up_read(&shrinker_rwsem);
akpm@osdl.orgb15e0902005-06-21 17:14:35 -0700240 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700241}
242
243/* Called without lock on whether page is mapped, so answer is unstable */
244static inline int page_mapping_inuse(struct page *page)
245{
246 struct address_space *mapping;
247
248 /* Page is in somebody's page tables. */
249 if (page_mapped(page))
250 return 1;
251
252 /* Be more reluctant to reclaim swapcache than pagecache */
253 if (PageSwapCache(page))
254 return 1;
255
256 mapping = page_mapping(page);
257 if (!mapping)
258 return 0;
259
260 /* File is mmap'd by somebody? */
261 return mapping_mapped(mapping);
262}
263
264static inline int is_page_cache_freeable(struct page *page)
265{
266 return page_count(page) - !!PagePrivate(page) == 2;
267}
268
269static int may_write_to_queue(struct backing_dev_info *bdi)
270{
271 if (current_is_kswapd())
272 return 1;
273 if (current_is_pdflush()) /* This is unlikely, but why not... */
274 return 1;
275 if (!bdi_write_congested(bdi))
276 return 1;
277 if (bdi == current->backing_dev_info)
278 return 1;
279 return 0;
280}
281
282/*
283 * We detected a synchronous write error writing a page out. Probably
284 * -ENOSPC. We need to propagate that into the address_space for a subsequent
285 * fsync(), msync() or close().
286 *
287 * The tricky part is that after writepage we cannot touch the mapping: nothing
288 * prevents it from being freed up. But we have a ref on the page and once
289 * that page is locked, the mapping is pinned.
290 *
291 * We're allowed to run sleeping lock_page() here because we know the caller has
292 * __GFP_FS.
293 */
294static void handle_write_error(struct address_space *mapping,
295 struct page *page, int error)
296{
297 lock_page(page);
298 if (page_mapping(page) == mapping) {
299 if (error == -ENOSPC)
300 set_bit(AS_ENOSPC, &mapping->flags);
301 else
302 set_bit(AS_EIO, &mapping->flags);
303 }
304 unlock_page(page);
305}
306
307/*
308 * pageout is called by shrink_list() for each dirty page. Calls ->writepage().
309 */
310static pageout_t pageout(struct page *page, struct address_space *mapping)
311{
312 /*
313 * If the page is dirty, only perform writeback if that write
314 * will be non-blocking. To prevent this allocation from being
315 * stalled by pagecache activity. But note that there may be
316 * stalls if we need to run get_block(). We could test
317 * PagePrivate for that.
318 *
319 * If this process is currently in generic_file_write() against
320 * this page's queue, we can perform writeback even if that
321 * will block.
322 *
323 * If the page is swapcache, write it back even if that would
324 * block, for some throttling. This happens by accident, because
325 * swap_backing_dev_info is bust: it doesn't reflect the
326 * congestion state of the swapdevs. Easy to fix, if needed.
327 * See swapfile.c:page_queue_congested().
328 */
329 if (!is_page_cache_freeable(page))
330 return PAGE_KEEP;
331 if (!mapping) {
332 /*
333 * Some data journaling orphaned pages can have
334 * page->mapping == NULL while being dirty with clean buffers.
335 */
akpm@osdl.org323aca62005-04-16 15:24:06 -0700336 if (PagePrivate(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700337 if (try_to_free_buffers(page)) {
338 ClearPageDirty(page);
339 printk("%s: orphaned page\n", __FUNCTION__);
340 return PAGE_CLEAN;
341 }
342 }
343 return PAGE_KEEP;
344 }
345 if (mapping->a_ops->writepage == NULL)
346 return PAGE_ACTIVATE;
347 if (!may_write_to_queue(mapping->backing_dev_info))
348 return PAGE_KEEP;
349
350 if (clear_page_dirty_for_io(page)) {
351 int res;
352 struct writeback_control wbc = {
353 .sync_mode = WB_SYNC_NONE,
354 .nr_to_write = SWAP_CLUSTER_MAX,
355 .nonblocking = 1,
356 .for_reclaim = 1,
357 };
358
359 SetPageReclaim(page);
360 res = mapping->a_ops->writepage(page, &wbc);
361 if (res < 0)
362 handle_write_error(mapping, page, res);
Zach Brown994fc28c2005-12-15 14:28:17 -0800363 if (res == AOP_WRITEPAGE_ACTIVATE) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364 ClearPageReclaim(page);
365 return PAGE_ACTIVATE;
366 }
367 if (!PageWriteback(page)) {
368 /* synchronous write or broken a_ops? */
369 ClearPageReclaim(page);
370 }
371
372 return PAGE_SUCCESS;
373 }
374
375 return PAGE_CLEAN;
376}
377
378/*
379 * shrink_list adds the number of reclaimed pages to sc->nr_reclaimed
380 */
381static int shrink_list(struct list_head *page_list, struct scan_control *sc)
382{
383 LIST_HEAD(ret_pages);
384 struct pagevec freed_pvec;
385 int pgactivate = 0;
386 int reclaimed = 0;
387
388 cond_resched();
389
390 pagevec_init(&freed_pvec, 1);
391 while (!list_empty(page_list)) {
392 struct address_space *mapping;
393 struct page *page;
394 int may_enter_fs;
395 int referenced;
396
397 cond_resched();
398
399 page = lru_to_page(page_list);
400 list_del(&page->lru);
401
402 if (TestSetPageLocked(page))
403 goto keep;
404
405 BUG_ON(PageActive(page));
406
407 sc->nr_scanned++;
408 /* Double the slab pressure for mapped and swapcache pages */
409 if (page_mapped(page) || PageSwapCache(page))
410 sc->nr_scanned++;
411
412 if (PageWriteback(page))
413 goto keep_locked;
414
Rik van Rielf7b7fd82005-11-28 13:44:07 -0800415 referenced = page_referenced(page, 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700416 /* In active use or really unfreeable? Activate it. */
417 if (referenced && page_mapping_inuse(page))
418 goto activate_locked;
419
420#ifdef CONFIG_SWAP
421 /*
422 * Anonymous process memory has backing store?
423 * Try to allocate it some swap space here.
424 */
Lee Schermerhornc3400102005-10-29 18:15:51 -0700425 if (PageAnon(page) && !PageSwapCache(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 if (!add_to_swap(page))
427 goto activate_locked;
428 }
429#endif /* CONFIG_SWAP */
430
431 mapping = page_mapping(page);
432 may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
433 (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
434
435 /*
436 * The page is mapped into the page tables of one or more
437 * processes. Try to unmap it here.
438 */
439 if (page_mapped(page) && mapping) {
440 switch (try_to_unmap(page)) {
441 case SWAP_FAIL:
442 goto activate_locked;
443 case SWAP_AGAIN:
444 goto keep_locked;
445 case SWAP_SUCCESS:
446 ; /* try to free the page below */
447 }
448 }
449
450 if (PageDirty(page)) {
451 if (referenced)
452 goto keep_locked;
453 if (!may_enter_fs)
454 goto keep_locked;
455 if (laptop_mode && !sc->may_writepage)
456 goto keep_locked;
457
458 /* Page is dirty, try to write it out here */
459 switch(pageout(page, mapping)) {
460 case PAGE_KEEP:
461 goto keep_locked;
462 case PAGE_ACTIVATE:
463 goto activate_locked;
464 case PAGE_SUCCESS:
465 if (PageWriteback(page) || PageDirty(page))
466 goto keep;
467 /*
468 * A synchronous write - probably a ramdisk. Go
469 * ahead and try to reclaim the page.
470 */
471 if (TestSetPageLocked(page))
472 goto keep;
473 if (PageDirty(page) || PageWriteback(page))
474 goto keep_locked;
475 mapping = page_mapping(page);
476 case PAGE_CLEAN:
477 ; /* try to free the page below */
478 }
479 }
480
481 /*
482 * If the page has buffers, try to free the buffer mappings
483 * associated with this page. If we succeed we try to free
484 * the page as well.
485 *
486 * We do this even if the page is PageDirty().
487 * try_to_release_page() does not perform I/O, but it is
488 * possible for a page to have PageDirty set, but it is actually
489 * clean (all its buffers are clean). This happens if the
490 * buffers were written out directly, with submit_bh(). ext3
491 * will do this, as well as the blockdev mapping.
492 * try_to_release_page() will discover that cleanness and will
493 * drop the buffers and mark the page clean - it can be freed.
494 *
495 * Rarely, pages can have buffers and no ->mapping. These are
496 * the pages which were not successfully invalidated in
497 * truncate_complete_page(). We try to drop those buffers here
498 * and if that worked, and the page is no longer mapped into
499 * process address space (page_count == 1) it can be freed.
500 * Otherwise, leave the page on the LRU so it is swappable.
501 */
502 if (PagePrivate(page)) {
503 if (!try_to_release_page(page, sc->gfp_mask))
504 goto activate_locked;
505 if (!mapping && page_count(page) == 1)
506 goto free_it;
507 }
508
509 if (!mapping)
510 goto keep_locked; /* truncate got there first */
511
512 write_lock_irq(&mapping->tree_lock);
513
514 /*
515 * The non-racy check for busy page. It is critical to check
516 * PageDirty _after_ making sure that the page is freeable and
517 * not in use by anybody. (pagecache + us == 2)
518 */
Linus Torvalds3d806362005-10-16 17:36:06 -0700519 if (unlikely(page_count(page) != 2))
520 goto cannot_free;
521 smp_rmb();
522 if (unlikely(PageDirty(page)))
523 goto cannot_free;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700524
525#ifdef CONFIG_SWAP
526 if (PageSwapCache(page)) {
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700527 swp_entry_t swap = { .val = page_private(page) };
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 __delete_from_swap_cache(page);
529 write_unlock_irq(&mapping->tree_lock);
530 swap_free(swap);
531 __put_page(page); /* The pagecache ref */
532 goto free_it;
533 }
534#endif /* CONFIG_SWAP */
535
536 __remove_from_page_cache(page);
537 write_unlock_irq(&mapping->tree_lock);
538 __put_page(page);
539
540free_it:
541 unlock_page(page);
542 reclaimed++;
543 if (!pagevec_add(&freed_pvec, page))
544 __pagevec_release_nonlru(&freed_pvec);
545 continue;
546
Linus Torvalds3d806362005-10-16 17:36:06 -0700547cannot_free:
548 write_unlock_irq(&mapping->tree_lock);
549 goto keep_locked;
550
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551activate_locked:
552 SetPageActive(page);
553 pgactivate++;
554keep_locked:
555 unlock_page(page);
556keep:
557 list_add(&page->lru, &ret_pages);
558 BUG_ON(PageLRU(page));
559 }
560 list_splice(&ret_pages, page_list);
561 if (pagevec_count(&freed_pvec))
562 __pagevec_release_nonlru(&freed_pvec);
563 mod_page_state(pgactivate, pgactivate);
564 sc->nr_reclaimed += reclaimed;
565 return reclaimed;
566}
567
568/*
569 * zone->lru_lock is heavily contended. Some of the functions that
570 * shrink the lists perform better by taking out a batch of pages
571 * and working on them outside the LRU lock.
572 *
573 * For pagecache intensive workloads, this function is the hottest
574 * spot in the kernel (apart from copy_*_user functions).
575 *
576 * Appropriate locks must be held before calling this function.
577 *
578 * @nr_to_scan: The number of pages to look through on the list.
579 * @src: The LRU list to pull pages off.
580 * @dst: The temp list to put pages on to.
581 * @scanned: The number of pages that were scanned.
582 *
583 * returns how many pages were moved onto *@dst.
584 */
585static int isolate_lru_pages(int nr_to_scan, struct list_head *src,
586 struct list_head *dst, int *scanned)
587{
588 int nr_taken = 0;
589 struct page *page;
590 int scan = 0;
591
592 while (scan++ < nr_to_scan && !list_empty(src)) {
593 page = lru_to_page(src);
594 prefetchw_prev_lru_page(page, src, flags);
595
Christoph Lameter21eac812006-01-08 01:00:45 -0800596 switch (__isolate_lru_page(page)) {
597 case 1:
598 /* Succeeded to isolate page */
599 list_move(&page->lru, dst);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700600 nr_taken++;
Christoph Lameter21eac812006-01-08 01:00:45 -0800601 break;
602 case -ENOENT:
603 /* Not possible to isolate */
604 list_move(&page->lru, src);
605 break;
606 default:
607 BUG();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700608 }
609 }
610
611 *scanned = scan;
612 return nr_taken;
613}
614
Christoph Lameter21eac812006-01-08 01:00:45 -0800615static void lru_add_drain_per_cpu(void *dummy)
616{
617 lru_add_drain();
618}
619
620/*
621 * Isolate one page from the LRU lists and put it on the
622 * indicated list. Do necessary cache draining if the
623 * page is not on the LRU lists yet.
624 *
625 * Result:
626 * 0 = page not on LRU list
627 * 1 = page removed from LRU list and added to the specified list.
628 * -ENOENT = page is being freed elsewhere.
629 */
630int isolate_lru_page(struct page *page)
631{
632 int rc = 0;
633 struct zone *zone = page_zone(page);
634
635redo:
636 spin_lock_irq(&zone->lru_lock);
637 rc = __isolate_lru_page(page);
638 if (rc == 1) {
639 if (PageActive(page))
640 del_page_from_active_list(zone, page);
641 else
642 del_page_from_inactive_list(zone, page);
643 }
644 spin_unlock_irq(&zone->lru_lock);
645 if (rc == 0) {
646 /*
647 * Maybe this page is still waiting for a cpu to drain it
648 * from one of the lru lists?
649 */
650 rc = schedule_on_each_cpu(lru_add_drain_per_cpu, NULL);
651 if (rc == 0 && PageLRU(page))
652 goto redo;
653 }
654 return rc;
655}
656
Linus Torvalds1da177e2005-04-16 15:20:36 -0700657/*
658 * shrink_cache() adds the number of pages reclaimed to sc->nr_reclaimed
659 */
660static void shrink_cache(struct zone *zone, struct scan_control *sc)
661{
662 LIST_HEAD(page_list);
663 struct pagevec pvec;
664 int max_scan = sc->nr_to_scan;
665
666 pagevec_init(&pvec, 1);
667
668 lru_add_drain();
669 spin_lock_irq(&zone->lru_lock);
670 while (max_scan > 0) {
671 struct page *page;
672 int nr_taken;
673 int nr_scan;
674 int nr_freed;
675
676 nr_taken = isolate_lru_pages(sc->swap_cluster_max,
677 &zone->inactive_list,
678 &page_list, &nr_scan);
679 zone->nr_inactive -= nr_taken;
680 zone->pages_scanned += nr_scan;
681 spin_unlock_irq(&zone->lru_lock);
682
683 if (nr_taken == 0)
684 goto done;
685
686 max_scan -= nr_scan;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687 nr_freed = shrink_list(&page_list, sc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700688
Nick Piggina74609f2006-01-06 00:11:20 -0800689 local_irq_disable();
690 if (current_is_kswapd()) {
691 __mod_page_state_zone(zone, pgscan_kswapd, nr_scan);
692 __mod_page_state(kswapd_steal, nr_freed);
693 } else
694 __mod_page_state_zone(zone, pgscan_direct, nr_scan);
695 __mod_page_state_zone(zone, pgsteal, nr_freed);
696
697 spin_lock(&zone->lru_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700698 /*
699 * Put back any unfreeable pages.
700 */
701 while (!list_empty(&page_list)) {
702 page = lru_to_page(&page_list);
703 if (TestSetPageLRU(page))
704 BUG();
705 list_del(&page->lru);
706 if (PageActive(page))
707 add_page_to_active_list(zone, page);
708 else
709 add_page_to_inactive_list(zone, page);
710 if (!pagevec_add(&pvec, page)) {
711 spin_unlock_irq(&zone->lru_lock);
712 __pagevec_release(&pvec);
713 spin_lock_irq(&zone->lru_lock);
714 }
715 }
716 }
717 spin_unlock_irq(&zone->lru_lock);
718done:
719 pagevec_release(&pvec);
720}
721
Christoph Lameter21eac812006-01-08 01:00:45 -0800722static inline void move_to_lru(struct page *page)
723{
724 list_del(&page->lru);
725 if (PageActive(page)) {
726 /*
727 * lru_cache_add_active checks that
728 * the PG_active bit is off.
729 */
730 ClearPageActive(page);
731 lru_cache_add_active(page);
732 } else {
733 lru_cache_add(page);
734 }
735 put_page(page);
736}
737
738/*
739 * Add isolated pages on the list back to the LRU
740 *
741 * returns the number of pages put back.
742 */
743int putback_lru_pages(struct list_head *l)
744{
745 struct page *page;
746 struct page *page2;
747 int count = 0;
748
749 list_for_each_entry_safe(page, page2, l, lru) {
750 move_to_lru(page);
751 count++;
752 }
753 return count;
754}
755
Linus Torvalds1da177e2005-04-16 15:20:36 -0700756/*
757 * This moves pages from the active list to the inactive list.
758 *
759 * We move them the other way if the page is referenced by one or more
760 * processes, from rmap.
761 *
762 * If the pages are mostly unmapped, the processing is fast and it is
763 * appropriate to hold zone->lru_lock across the whole operation. But if
764 * the pages are mapped, the processing is slow (page_referenced()) so we
765 * should drop zone->lru_lock around each page. It's impossible to balance
766 * this, so instead we remove the pages from the LRU while processing them.
767 * It is safe to rely on PG_active against the non-LRU pages in here because
768 * nobody will play with that bit on a non-LRU page.
769 *
770 * The downside is that we have to touch page->_count against each page.
771 * But we had to alter page->flags anyway.
772 */
773static void
774refill_inactive_zone(struct zone *zone, struct scan_control *sc)
775{
776 int pgmoved;
777 int pgdeactivate = 0;
778 int pgscanned;
779 int nr_pages = sc->nr_to_scan;
780 LIST_HEAD(l_hold); /* The pages which were snipped off */
781 LIST_HEAD(l_inactive); /* Pages to go onto the inactive_list */
782 LIST_HEAD(l_active); /* Pages to go onto the active_list */
783 struct page *page;
784 struct pagevec pvec;
785 int reclaim_mapped = 0;
786 long mapped_ratio;
787 long distress;
788 long swap_tendency;
789
790 lru_add_drain();
791 spin_lock_irq(&zone->lru_lock);
792 pgmoved = isolate_lru_pages(nr_pages, &zone->active_list,
793 &l_hold, &pgscanned);
794 zone->pages_scanned += pgscanned;
795 zone->nr_active -= pgmoved;
796 spin_unlock_irq(&zone->lru_lock);
797
798 /*
799 * `distress' is a measure of how much trouble we're having reclaiming
800 * pages. 0 -> no problems. 100 -> great trouble.
801 */
802 distress = 100 >> zone->prev_priority;
803
804 /*
805 * The point of this algorithm is to decide when to start reclaiming
806 * mapped memory instead of just pagecache. Work out how much memory
807 * is mapped.
808 */
809 mapped_ratio = (sc->nr_mapped * 100) / total_memory;
810
811 /*
812 * Now decide how much we really want to unmap some pages. The mapped
813 * ratio is downgraded - just because there's a lot of mapped memory
814 * doesn't necessarily mean that page reclaim isn't succeeding.
815 *
816 * The distress ratio is important - we don't want to start going oom.
817 *
818 * A 100% value of vm_swappiness overrides this algorithm altogether.
819 */
820 swap_tendency = mapped_ratio / 2 + distress + vm_swappiness;
821
822 /*
823 * Now use this metric to decide whether to start moving mapped memory
824 * onto the inactive list.
825 */
826 if (swap_tendency >= 100)
827 reclaim_mapped = 1;
828
829 while (!list_empty(&l_hold)) {
830 cond_resched();
831 page = lru_to_page(&l_hold);
832 list_del(&page->lru);
833 if (page_mapped(page)) {
834 if (!reclaim_mapped ||
835 (total_swap_pages == 0 && PageAnon(page)) ||
Rik van Rielf7b7fd82005-11-28 13:44:07 -0800836 page_referenced(page, 0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837 list_add(&page->lru, &l_active);
838 continue;
839 }
840 }
841 list_add(&page->lru, &l_inactive);
842 }
843
844 pagevec_init(&pvec, 1);
845 pgmoved = 0;
846 spin_lock_irq(&zone->lru_lock);
847 while (!list_empty(&l_inactive)) {
848 page = lru_to_page(&l_inactive);
849 prefetchw_prev_lru_page(page, &l_inactive, flags);
850 if (TestSetPageLRU(page))
851 BUG();
852 if (!TestClearPageActive(page))
853 BUG();
854 list_move(&page->lru, &zone->inactive_list);
855 pgmoved++;
856 if (!pagevec_add(&pvec, page)) {
857 zone->nr_inactive += pgmoved;
858 spin_unlock_irq(&zone->lru_lock);
859 pgdeactivate += pgmoved;
860 pgmoved = 0;
861 if (buffer_heads_over_limit)
862 pagevec_strip(&pvec);
863 __pagevec_release(&pvec);
864 spin_lock_irq(&zone->lru_lock);
865 }
866 }
867 zone->nr_inactive += pgmoved;
868 pgdeactivate += pgmoved;
869 if (buffer_heads_over_limit) {
870 spin_unlock_irq(&zone->lru_lock);
871 pagevec_strip(&pvec);
872 spin_lock_irq(&zone->lru_lock);
873 }
874
875 pgmoved = 0;
876 while (!list_empty(&l_active)) {
877 page = lru_to_page(&l_active);
878 prefetchw_prev_lru_page(page, &l_active, flags);
879 if (TestSetPageLRU(page))
880 BUG();
881 BUG_ON(!PageActive(page));
882 list_move(&page->lru, &zone->active_list);
883 pgmoved++;
884 if (!pagevec_add(&pvec, page)) {
885 zone->nr_active += pgmoved;
886 pgmoved = 0;
887 spin_unlock_irq(&zone->lru_lock);
888 __pagevec_release(&pvec);
889 spin_lock_irq(&zone->lru_lock);
890 }
891 }
892 zone->nr_active += pgmoved;
Nick Piggina74609f2006-01-06 00:11:20 -0800893 spin_unlock(&zone->lru_lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700894
Nick Piggina74609f2006-01-06 00:11:20 -0800895 __mod_page_state_zone(zone, pgrefill, pgscanned);
896 __mod_page_state(pgdeactivate, pgdeactivate);
897 local_irq_enable();
898
899 pagevec_release(&pvec);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700900}
901
902/*
903 * This is a basic per-zone page freer. Used by both kswapd and direct reclaim.
904 */
905static void
906shrink_zone(struct zone *zone, struct scan_control *sc)
907{
908 unsigned long nr_active;
909 unsigned long nr_inactive;
910
Martin Hicks53e9a612005-09-03 15:54:51 -0700911 atomic_inc(&zone->reclaim_in_progress);
912
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913 /*
914 * Add one to `nr_to_scan' just to make sure that the kernel will
915 * slowly sift through the active list.
916 */
917 zone->nr_scan_active += (zone->nr_active >> sc->priority) + 1;
918 nr_active = zone->nr_scan_active;
919 if (nr_active >= sc->swap_cluster_max)
920 zone->nr_scan_active = 0;
921 else
922 nr_active = 0;
923
924 zone->nr_scan_inactive += (zone->nr_inactive >> sc->priority) + 1;
925 nr_inactive = zone->nr_scan_inactive;
926 if (nr_inactive >= sc->swap_cluster_max)
927 zone->nr_scan_inactive = 0;
928 else
929 nr_inactive = 0;
930
Linus Torvalds1da177e2005-04-16 15:20:36 -0700931 while (nr_active || nr_inactive) {
932 if (nr_active) {
933 sc->nr_to_scan = min(nr_active,
934 (unsigned long)sc->swap_cluster_max);
935 nr_active -= sc->nr_to_scan;
936 refill_inactive_zone(zone, sc);
937 }
938
939 if (nr_inactive) {
940 sc->nr_to_scan = min(nr_inactive,
941 (unsigned long)sc->swap_cluster_max);
942 nr_inactive -= sc->nr_to_scan;
943 shrink_cache(zone, sc);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700944 }
945 }
946
947 throttle_vm_writeout();
Martin Hicks53e9a612005-09-03 15:54:51 -0700948
949 atomic_dec(&zone->reclaim_in_progress);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700950}
951
952/*
953 * This is the direct reclaim path, for page-allocating processes. We only
954 * try to reclaim pages from zones which will satisfy the caller's allocation
955 * request.
956 *
957 * We reclaim from a zone even if that zone is over pages_high. Because:
958 * a) The caller may be trying to free *extra* pages to satisfy a higher-order
959 * allocation or
960 * b) The zones may be over pages_high but they must go *over* pages_high to
961 * satisfy the `incremental min' zone defense algorithm.
962 *
963 * Returns the number of reclaimed pages.
964 *
965 * If a zone is deemed to be full of pinned pages then just give it a light
966 * scan then give up on it.
967 */
968static void
969shrink_caches(struct zone **zones, struct scan_control *sc)
970{
971 int i;
972
973 for (i = 0; zones[i] != NULL; i++) {
974 struct zone *zone = zones[i];
975
Con Kolivasf3fe6512006-01-06 00:11:15 -0800976 if (!populated_zone(zone))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700977 continue;
978
Paul Jackson9bf22292005-09-06 15:18:12 -0700979 if (!cpuset_zone_allowed(zone, __GFP_HARDWALL))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980 continue;
981
982 zone->temp_priority = sc->priority;
983 if (zone->prev_priority > sc->priority)
984 zone->prev_priority = sc->priority;
985
986 if (zone->all_unreclaimable && sc->priority != DEF_PRIORITY)
987 continue; /* Let kswapd poll it */
988
989 shrink_zone(zone, sc);
990 }
991}
992
993/*
994 * This is the main entry point to direct page reclaim.
995 *
996 * If a full scan of the inactive list fails to free enough memory then we
997 * are "out of memory" and something needs to be killed.
998 *
999 * If the caller is !__GFP_FS then the probability of a failure is reasonably
1000 * high - the zone may be full of dirty or under-writeback pages, which this
1001 * caller can't do much about. We kick pdflush and take explicit naps in the
1002 * hope that some of these pages can be written. But if the allocating task
1003 * holds filesystem locks which prevent writeout this might not work, and the
1004 * allocation attempt will fail.
1005 */
Al Viro6daa0e22005-10-21 03:18:50 -04001006int try_to_free_pages(struct zone **zones, gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001007{
1008 int priority;
1009 int ret = 0;
1010 int total_scanned = 0, total_reclaimed = 0;
1011 struct reclaim_state *reclaim_state = current->reclaim_state;
1012 struct scan_control sc;
1013 unsigned long lru_pages = 0;
1014 int i;
1015
1016 sc.gfp_mask = gfp_mask;
1017 sc.may_writepage = 0;
1018
1019 inc_page_state(allocstall);
1020
1021 for (i = 0; zones[i] != NULL; i++) {
1022 struct zone *zone = zones[i];
1023
Paul Jackson9bf22292005-09-06 15:18:12 -07001024 if (!cpuset_zone_allowed(zone, __GFP_HARDWALL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001025 continue;
1026
1027 zone->temp_priority = DEF_PRIORITY;
1028 lru_pages += zone->nr_active + zone->nr_inactive;
1029 }
1030
1031 for (priority = DEF_PRIORITY; priority >= 0; priority--) {
1032 sc.nr_mapped = read_page_state(nr_mapped);
1033 sc.nr_scanned = 0;
1034 sc.nr_reclaimed = 0;
1035 sc.priority = priority;
1036 sc.swap_cluster_max = SWAP_CLUSTER_MAX;
Rik van Rielf7b7fd82005-11-28 13:44:07 -08001037 if (!priority)
1038 disable_swap_token();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001039 shrink_caches(zones, &sc);
1040 shrink_slab(sc.nr_scanned, gfp_mask, lru_pages);
1041 if (reclaim_state) {
1042 sc.nr_reclaimed += reclaim_state->reclaimed_slab;
1043 reclaim_state->reclaimed_slab = 0;
1044 }
1045 total_scanned += sc.nr_scanned;
1046 total_reclaimed += sc.nr_reclaimed;
1047 if (total_reclaimed >= sc.swap_cluster_max) {
1048 ret = 1;
1049 goto out;
1050 }
1051
1052 /*
1053 * Try to write back as many pages as we just scanned. This
1054 * tends to cause slow streaming writers to write data to the
1055 * disk smoothly, at the dirtying rate, which is nice. But
1056 * that's undesirable in laptop mode, where we *want* lumpy
1057 * writeout. So in laptop mode, write out the whole world.
1058 */
1059 if (total_scanned > sc.swap_cluster_max + sc.swap_cluster_max/2) {
Pekka J Enberg687a21c2005-06-28 20:44:55 -07001060 wakeup_pdflush(laptop_mode ? 0 : total_scanned);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001061 sc.may_writepage = 1;
1062 }
1063
1064 /* Take a nap, wait for some writeback to complete */
1065 if (sc.nr_scanned && priority < DEF_PRIORITY - 2)
1066 blk_congestion_wait(WRITE, HZ/10);
1067 }
1068out:
1069 for (i = 0; zones[i] != 0; i++) {
1070 struct zone *zone = zones[i];
1071
Paul Jackson9bf22292005-09-06 15:18:12 -07001072 if (!cpuset_zone_allowed(zone, __GFP_HARDWALL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001073 continue;
1074
1075 zone->prev_priority = zone->temp_priority;
1076 }
1077 return ret;
1078}
1079
1080/*
1081 * For kswapd, balance_pgdat() will work across all this node's zones until
1082 * they are all at pages_high.
1083 *
1084 * If `nr_pages' is non-zero then it is the number of pages which are to be
1085 * reclaimed, regardless of the zone occupancies. This is a software suspend
1086 * special.
1087 *
1088 * Returns the number of pages which were actually freed.
1089 *
1090 * There is special handling here for zones which are full of pinned pages.
1091 * This can happen if the pages are all mlocked, or if they are all used by
1092 * device drivers (say, ZONE_DMA). Or if they are all in use by hugetlb.
1093 * What we do is to detect the case where all pages in the zone have been
1094 * scanned twice and there has been zero successful reclaim. Mark the zone as
1095 * dead and from now on, only perform a short scan. Basically we're polling
1096 * the zone for when the problem goes away.
1097 *
1098 * kswapd scans the zones in the highmem->normal->dma direction. It skips
1099 * zones which have free_pages > pages_high, but once a zone is found to have
1100 * free_pages <= pages_high, we scan that zone and the lower zones regardless
1101 * of the number of free pages in the lower zones. This interoperates with
1102 * the page allocator fallback scheme to ensure that aging of pages is balanced
1103 * across the zones.
1104 */
1105static int balance_pgdat(pg_data_t *pgdat, int nr_pages, int order)
1106{
1107 int to_free = nr_pages;
1108 int all_zones_ok;
1109 int priority;
1110 int i;
1111 int total_scanned, total_reclaimed;
1112 struct reclaim_state *reclaim_state = current->reclaim_state;
1113 struct scan_control sc;
1114
1115loop_again:
1116 total_scanned = 0;
1117 total_reclaimed = 0;
1118 sc.gfp_mask = GFP_KERNEL;
1119 sc.may_writepage = 0;
1120 sc.nr_mapped = read_page_state(nr_mapped);
1121
1122 inc_page_state(pageoutrun);
1123
1124 for (i = 0; i < pgdat->nr_zones; i++) {
1125 struct zone *zone = pgdat->node_zones + i;
1126
1127 zone->temp_priority = DEF_PRIORITY;
1128 }
1129
1130 for (priority = DEF_PRIORITY; priority >= 0; priority--) {
1131 int end_zone = 0; /* Inclusive. 0 = ZONE_DMA */
1132 unsigned long lru_pages = 0;
1133
Rik van Rielf7b7fd82005-11-28 13:44:07 -08001134 /* The swap token gets in the way of swapout... */
1135 if (!priority)
1136 disable_swap_token();
1137
Linus Torvalds1da177e2005-04-16 15:20:36 -07001138 all_zones_ok = 1;
1139
1140 if (nr_pages == 0) {
1141 /*
1142 * Scan in the highmem->dma direction for the highest
1143 * zone which needs scanning
1144 */
1145 for (i = pgdat->nr_zones - 1; i >= 0; i--) {
1146 struct zone *zone = pgdat->node_zones + i;
1147
Con Kolivasf3fe6512006-01-06 00:11:15 -08001148 if (!populated_zone(zone))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001149 continue;
1150
1151 if (zone->all_unreclaimable &&
1152 priority != DEF_PRIORITY)
1153 continue;
1154
1155 if (!zone_watermark_ok(zone, order,
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001156 zone->pages_high, 0, 0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001157 end_zone = i;
1158 goto scan;
1159 }
1160 }
1161 goto out;
1162 } else {
1163 end_zone = pgdat->nr_zones - 1;
1164 }
1165scan:
1166 for (i = 0; i <= end_zone; i++) {
1167 struct zone *zone = pgdat->node_zones + i;
1168
1169 lru_pages += zone->nr_active + zone->nr_inactive;
1170 }
1171
1172 /*
1173 * Now scan the zone in the dma->highmem direction, stopping
1174 * at the last zone which needs scanning.
1175 *
1176 * We do this because the page allocator works in the opposite
1177 * direction. This prevents the page allocator from allocating
1178 * pages behind kswapd's direction of progress, which would
1179 * cause too much scanning of the lower zones.
1180 */
1181 for (i = 0; i <= end_zone; i++) {
1182 struct zone *zone = pgdat->node_zones + i;
akpm@osdl.orgb15e0902005-06-21 17:14:35 -07001183 int nr_slab;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001184
Con Kolivasf3fe6512006-01-06 00:11:15 -08001185 if (!populated_zone(zone))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001186 continue;
1187
1188 if (zone->all_unreclaimable && priority != DEF_PRIORITY)
1189 continue;
1190
1191 if (nr_pages == 0) { /* Not software suspend */
1192 if (!zone_watermark_ok(zone, order,
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001193 zone->pages_high, end_zone, 0))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001194 all_zones_ok = 0;
1195 }
1196 zone->temp_priority = priority;
1197 if (zone->prev_priority > priority)
1198 zone->prev_priority = priority;
1199 sc.nr_scanned = 0;
1200 sc.nr_reclaimed = 0;
1201 sc.priority = priority;
1202 sc.swap_cluster_max = nr_pages? nr_pages : SWAP_CLUSTER_MAX;
Martin Hicks1e7e5a92005-06-21 17:14:43 -07001203 atomic_inc(&zone->reclaim_in_progress);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001204 shrink_zone(zone, &sc);
Martin Hicks1e7e5a92005-06-21 17:14:43 -07001205 atomic_dec(&zone->reclaim_in_progress);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001206 reclaim_state->reclaimed_slab = 0;
akpm@osdl.orgb15e0902005-06-21 17:14:35 -07001207 nr_slab = shrink_slab(sc.nr_scanned, GFP_KERNEL,
1208 lru_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001209 sc.nr_reclaimed += reclaim_state->reclaimed_slab;
1210 total_reclaimed += sc.nr_reclaimed;
1211 total_scanned += sc.nr_scanned;
1212 if (zone->all_unreclaimable)
1213 continue;
akpm@osdl.orgb15e0902005-06-21 17:14:35 -07001214 if (nr_slab == 0 && zone->pages_scanned >=
1215 (zone->nr_active + zone->nr_inactive) * 4)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001216 zone->all_unreclaimable = 1;
1217 /*
1218 * If we've done a decent amount of scanning and
1219 * the reclaim ratio is low, start doing writepage
1220 * even in laptop mode
1221 */
1222 if (total_scanned > SWAP_CLUSTER_MAX * 2 &&
1223 total_scanned > total_reclaimed+total_reclaimed/2)
1224 sc.may_writepage = 1;
1225 }
1226 if (nr_pages && to_free > total_reclaimed)
1227 continue; /* swsusp: need to do more work */
1228 if (all_zones_ok)
1229 break; /* kswapd: all done */
1230 /*
1231 * OK, kswapd is getting into trouble. Take a nap, then take
1232 * another pass across the zones.
1233 */
1234 if (total_scanned && priority < DEF_PRIORITY - 2)
1235 blk_congestion_wait(WRITE, HZ/10);
1236
1237 /*
1238 * We do this so kswapd doesn't build up large priorities for
1239 * example when it is freeing in parallel with allocators. It
1240 * matches the direct reclaim path behaviour in terms of impact
1241 * on zone->*_priority.
1242 */
1243 if ((total_reclaimed >= SWAP_CLUSTER_MAX) && (!nr_pages))
1244 break;
1245 }
1246out:
1247 for (i = 0; i < pgdat->nr_zones; i++) {
1248 struct zone *zone = pgdat->node_zones + i;
1249
1250 zone->prev_priority = zone->temp_priority;
1251 }
1252 if (!all_zones_ok) {
1253 cond_resched();
1254 goto loop_again;
1255 }
1256
1257 return total_reclaimed;
1258}
1259
1260/*
1261 * The background pageout daemon, started as a kernel thread
1262 * from the init process.
1263 *
1264 * This basically trickles out pages so that we have _some_
1265 * free memory available even if there is no other activity
1266 * that frees anything up. This is needed for things like routing
1267 * etc, where we otherwise might have all activity going on in
1268 * asynchronous contexts that cannot page things out.
1269 *
1270 * If there are applications that are active memory-allocators
1271 * (most normal use), this basically shouldn't matter.
1272 */
1273static int kswapd(void *p)
1274{
1275 unsigned long order;
1276 pg_data_t *pgdat = (pg_data_t*)p;
1277 struct task_struct *tsk = current;
1278 DEFINE_WAIT(wait);
1279 struct reclaim_state reclaim_state = {
1280 .reclaimed_slab = 0,
1281 };
1282 cpumask_t cpumask;
1283
1284 daemonize("kswapd%d", pgdat->node_id);
1285 cpumask = node_to_cpumask(pgdat->node_id);
1286 if (!cpus_empty(cpumask))
1287 set_cpus_allowed(tsk, cpumask);
1288 current->reclaim_state = &reclaim_state;
1289
1290 /*
1291 * Tell the memory management that we're a "memory allocator",
1292 * and that if we need more memory we should get access to it
1293 * regardless (see "__alloc_pages()"). "kswapd" should
1294 * never get caught in the normal page freeing logic.
1295 *
1296 * (Kswapd normally doesn't need memory anyway, but sometimes
1297 * you need a small amount of memory in order to be able to
1298 * page out something else, and this flag essentially protects
1299 * us from recursively trying to free more memory as we're
1300 * trying to free the first piece of memory in the first place).
1301 */
1302 tsk->flags |= PF_MEMALLOC|PF_KSWAPD;
1303
1304 order = 0;
1305 for ( ; ; ) {
1306 unsigned long new_order;
Christoph Lameter3e1d1d22005-06-24 23:13:50 -07001307
1308 try_to_freeze();
Linus Torvalds1da177e2005-04-16 15:20:36 -07001309
1310 prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
1311 new_order = pgdat->kswapd_max_order;
1312 pgdat->kswapd_max_order = 0;
1313 if (order < new_order) {
1314 /*
1315 * Don't sleep if someone wants a larger 'order'
1316 * allocation
1317 */
1318 order = new_order;
1319 } else {
1320 schedule();
1321 order = pgdat->kswapd_max_order;
1322 }
1323 finish_wait(&pgdat->kswapd_wait, &wait);
1324
1325 balance_pgdat(pgdat, 0, order);
1326 }
1327 return 0;
1328}
1329
1330/*
1331 * A zone is low on free memory, so wake its kswapd task to service it.
1332 */
1333void wakeup_kswapd(struct zone *zone, int order)
1334{
1335 pg_data_t *pgdat;
1336
Con Kolivasf3fe6512006-01-06 00:11:15 -08001337 if (!populated_zone(zone))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001338 return;
1339
1340 pgdat = zone->zone_pgdat;
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001341 if (zone_watermark_ok(zone, order, zone->pages_low, 0, 0))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001342 return;
1343 if (pgdat->kswapd_max_order < order)
1344 pgdat->kswapd_max_order = order;
Paul Jackson9bf22292005-09-06 15:18:12 -07001345 if (!cpuset_zone_allowed(zone, __GFP_HARDWALL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346 return;
Con Kolivas8d0986e2005-09-13 01:25:07 -07001347 if (!waitqueue_active(&pgdat->kswapd_wait))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001348 return;
Con Kolivas8d0986e2005-09-13 01:25:07 -07001349 wake_up_interruptible(&pgdat->kswapd_wait);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001350}
1351
1352#ifdef CONFIG_PM
1353/*
1354 * Try to free `nr_pages' of memory, system-wide. Returns the number of freed
1355 * pages.
1356 */
1357int shrink_all_memory(int nr_pages)
1358{
1359 pg_data_t *pgdat;
1360 int nr_to_free = nr_pages;
1361 int ret = 0;
1362 struct reclaim_state reclaim_state = {
1363 .reclaimed_slab = 0,
1364 };
1365
1366 current->reclaim_state = &reclaim_state;
1367 for_each_pgdat(pgdat) {
1368 int freed;
1369 freed = balance_pgdat(pgdat, nr_to_free, 0);
1370 ret += freed;
1371 nr_to_free -= freed;
1372 if (nr_to_free <= 0)
1373 break;
1374 }
1375 current->reclaim_state = NULL;
1376 return ret;
1377}
1378#endif
1379
1380#ifdef CONFIG_HOTPLUG_CPU
1381/* It's optimal to keep kswapds on the same CPUs as their memory, but
1382 not required for correctness. So if the last cpu in a node goes
1383 away, we get changed to run anywhere: as the first one comes back,
1384 restore their cpu bindings. */
1385static int __devinit cpu_callback(struct notifier_block *nfb,
1386 unsigned long action,
1387 void *hcpu)
1388{
1389 pg_data_t *pgdat;
1390 cpumask_t mask;
1391
1392 if (action == CPU_ONLINE) {
1393 for_each_pgdat(pgdat) {
1394 mask = node_to_cpumask(pgdat->node_id);
1395 if (any_online_cpu(mask) != NR_CPUS)
1396 /* One of our CPUs online: restore mask */
1397 set_cpus_allowed(pgdat->kswapd, mask);
1398 }
1399 }
1400 return NOTIFY_OK;
1401}
1402#endif /* CONFIG_HOTPLUG_CPU */
1403
1404static int __init kswapd_init(void)
1405{
1406 pg_data_t *pgdat;
1407 swap_setup();
1408 for_each_pgdat(pgdat)
1409 pgdat->kswapd
1410 = find_task_by_pid(kernel_thread(kswapd, pgdat, CLONE_KERNEL));
1411 total_memory = nr_free_pagecache_pages();
1412 hotcpu_notifier(cpu_callback, 0);
1413 return 0;
1414}
1415
1416module_init(kswapd_init)