blob: 153572fb60b8debb24359e195e7a53b50f8e1c37 [file] [log] [blame]
Christoph Lameterb20a3502006-03-22 00:09:12 -08001/*
2 * Memory Migration functionality - linux/mm/migration.c
3 *
4 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
5 *
6 * Page migration was first developed in the context of the memory hotplug
7 * project. The main authors of the migration code are:
8 *
9 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
10 * Hirokazu Takahashi <taka@valinux.co.jp>
11 * Dave Hansen <haveblue@us.ibm.com>
Christoph Lametercde53532008-07-04 09:59:22 -070012 * Christoph Lameter
Christoph Lameterb20a3502006-03-22 00:09:12 -080013 */
14
15#include <linux/migrate.h>
16#include <linux/module.h>
17#include <linux/swap.h>
Christoph Lameter06972122006-06-23 02:03:35 -070018#include <linux/swapops.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080019#include <linux/pagemap.h>
Christoph Lametere23ca002006-04-10 22:52:57 -070020#include <linux/buffer_head.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080021#include <linux/mm_inline.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070022#include <linux/nsproxy.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080023#include <linux/pagevec.h>
24#include <linux/rmap.h>
25#include <linux/topology.h>
26#include <linux/cpu.h>
27#include <linux/cpuset.h>
Christoph Lameter04e62a22006-06-23 02:03:38 -070028#include <linux/writeback.h>
Christoph Lameter742755a2006-06-23 02:03:55 -070029#include <linux/mempolicy.h>
30#include <linux/vmalloc.h>
David Quigley86c3a762006-06-23 02:04:02 -070031#include <linux/security.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080032#include <linux/memcontrol.h>
Adrian Bunk4f5ca262008-07-23 21:27:02 -070033#include <linux/syscalls.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080034
35#include "internal.h"
36
Christoph Lameterb20a3502006-03-22 00:09:12 -080037#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
38
39/*
40 * Isolate one page from the LRU lists. If successful put it onto
41 * the indicated list with elevated page count.
42 *
43 * Result:
44 * -EBUSY: page not on LRU list
45 * 0: page removed from LRU list and added to the specified list.
46 */
47int isolate_lru_page(struct page *page, struct list_head *pagelist)
48{
49 int ret = -EBUSY;
50
51 if (PageLRU(page)) {
52 struct zone *zone = page_zone(page);
53
54 spin_lock_irq(&zone->lru_lock);
KAMEZAWA Hiroyuki3dd9fe82007-07-26 10:41:08 -070055 if (PageLRU(page) && get_page_unless_zero(page)) {
Christoph Lameterb20a3502006-03-22 00:09:12 -080056 ret = 0;
Christoph Lameterb20a3502006-03-22 00:09:12 -080057 ClearPageLRU(page);
58 if (PageActive(page))
59 del_page_from_active_list(zone, page);
60 else
61 del_page_from_inactive_list(zone, page);
62 list_add_tail(&page->lru, pagelist);
63 }
64 spin_unlock_irq(&zone->lru_lock);
65 }
66 return ret;
67}
68
69/*
Christoph Lameter742755a2006-06-23 02:03:55 -070070 * migrate_prep() needs to be called before we start compiling a list of pages
71 * to be migrated using isolate_lru_page().
Christoph Lameterb20a3502006-03-22 00:09:12 -080072 */
73int migrate_prep(void)
74{
Christoph Lameterb20a3502006-03-22 00:09:12 -080075 /*
76 * Clear the LRU lists so pages can be isolated.
77 * Note that pages may be moved off the LRU after we have
78 * drained them. Those pages will fail to migrate like other
79 * pages that may be busy.
80 */
81 lru_add_drain_all();
82
83 return 0;
84}
85
86static inline void move_to_lru(struct page *page)
87{
Christoph Lameterb20a3502006-03-22 00:09:12 -080088 if (PageActive(page)) {
89 /*
90 * lru_cache_add_active checks that
91 * the PG_active bit is off.
92 */
93 ClearPageActive(page);
94 lru_cache_add_active(page);
95 } else {
96 lru_cache_add(page);
97 }
98 put_page(page);
99}
100
101/*
102 * Add isolated pages on the list back to the LRU.
103 *
104 * returns the number of pages put back.
105 */
106int putback_lru_pages(struct list_head *l)
107{
108 struct page *page;
109 struct page *page2;
110 int count = 0;
111
112 list_for_each_entry_safe(page, page2, l, lru) {
Christoph Lametere24f0b82006-06-23 02:03:51 -0700113 list_del(&page->lru);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800114 move_to_lru(page);
115 count++;
116 }
117 return count;
118}
119
Christoph Lameter06972122006-06-23 02:03:35 -0700120/*
121 * Restore a potential migration pte to a working pte entry
122 */
Christoph Lameter04e62a22006-06-23 02:03:38 -0700123static void remove_migration_pte(struct vm_area_struct *vma,
Christoph Lameter06972122006-06-23 02:03:35 -0700124 struct page *old, struct page *new)
125{
126 struct mm_struct *mm = vma->vm_mm;
127 swp_entry_t entry;
128 pgd_t *pgd;
129 pud_t *pud;
130 pmd_t *pmd;
131 pte_t *ptep, pte;
132 spinlock_t *ptl;
Christoph Lameter04e62a22006-06-23 02:03:38 -0700133 unsigned long addr = page_address_in_vma(new, vma);
134
135 if (addr == -EFAULT)
136 return;
Christoph Lameter06972122006-06-23 02:03:35 -0700137
138 pgd = pgd_offset(mm, addr);
139 if (!pgd_present(*pgd))
140 return;
141
142 pud = pud_offset(pgd, addr);
143 if (!pud_present(*pud))
144 return;
145
146 pmd = pmd_offset(pud, addr);
147 if (!pmd_present(*pmd))
148 return;
149
150 ptep = pte_offset_map(pmd, addr);
151
152 if (!is_swap_pte(*ptep)) {
153 pte_unmap(ptep);
154 return;
155 }
156
157 ptl = pte_lockptr(mm, pmd);
158 spin_lock(ptl);
159 pte = *ptep;
160 if (!is_swap_pte(pte))
161 goto out;
162
163 entry = pte_to_swp_entry(pte);
164
165 if (!is_migration_entry(entry) || migration_entry_to_page(entry) != old)
166 goto out;
167
Hugh Dickins98837c72008-03-04 14:29:06 -0800168 /*
169 * Yes, ignore the return value from a GFP_ATOMIC mem_cgroup_charge.
170 * Failure is not an option here: we're now expected to remove every
171 * migration pte, and will cause crashes otherwise. Normally this
172 * is not an issue: mem_cgroup_prepare_migration bumped up the old
173 * page_cgroup count for safety, that's now attached to the new page,
174 * so this charge should just be another incrementation of the count,
175 * to keep in balance with rmap.c's mem_cgroup_uncharging. But if
176 * there's been a force_empty, those reference counts may no longer
177 * be reliable, and this charge can actually fail: oh well, we don't
178 * make the situation any worse by proceeding as if it had succeeded.
179 */
180 mem_cgroup_charge(new, mm, GFP_ATOMIC);
181
Christoph Lameter06972122006-06-23 02:03:35 -0700182 get_page(new);
183 pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
184 if (is_write_migration_entry(entry))
185 pte = pte_mkwrite(pte);
KAMEZAWA Hiroyuki97ee0522007-10-16 01:25:43 -0700186 flush_cache_page(vma, addr, pte_pfn(pte));
Christoph Lameter06972122006-06-23 02:03:35 -0700187 set_pte_at(mm, addr, ptep, pte);
Christoph Lameter04e62a22006-06-23 02:03:38 -0700188
189 if (PageAnon(new))
190 page_add_anon_rmap(new, vma, addr);
191 else
192 page_add_file_rmap(new);
193
194 /* No need to invalidate - it was non-present before */
195 update_mmu_cache(vma, addr, pte);
Christoph Lameter04e62a22006-06-23 02:03:38 -0700196
Christoph Lameter06972122006-06-23 02:03:35 -0700197out:
198 pte_unmap_unlock(ptep, ptl);
199}
200
201/*
Christoph Lameter04e62a22006-06-23 02:03:38 -0700202 * Note that remove_file_migration_ptes will only work on regular mappings,
203 * Nonlinear mappings do not use migration entries.
204 */
205static void remove_file_migration_ptes(struct page *old, struct page *new)
206{
207 struct vm_area_struct *vma;
208 struct address_space *mapping = page_mapping(new);
209 struct prio_tree_iter iter;
210 pgoff_t pgoff = new->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
211
212 if (!mapping)
213 return;
214
215 spin_lock(&mapping->i_mmap_lock);
216
217 vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff)
218 remove_migration_pte(vma, old, new);
219
220 spin_unlock(&mapping->i_mmap_lock);
221}
222
223/*
Christoph Lameter06972122006-06-23 02:03:35 -0700224 * Must hold mmap_sem lock on at least one of the vmas containing
225 * the page so that the anon_vma cannot vanish.
226 */
Christoph Lameter04e62a22006-06-23 02:03:38 -0700227static void remove_anon_migration_ptes(struct page *old, struct page *new)
Christoph Lameter06972122006-06-23 02:03:35 -0700228{
229 struct anon_vma *anon_vma;
230 struct vm_area_struct *vma;
231 unsigned long mapping;
232
233 mapping = (unsigned long)new->mapping;
234
235 if (!mapping || (mapping & PAGE_MAPPING_ANON) == 0)
236 return;
237
238 /*
239 * We hold the mmap_sem lock. So no need to call page_lock_anon_vma.
240 */
241 anon_vma = (struct anon_vma *) (mapping - PAGE_MAPPING_ANON);
242 spin_lock(&anon_vma->lock);
243
244 list_for_each_entry(vma, &anon_vma->head, anon_vma_node)
Christoph Lameter04e62a22006-06-23 02:03:38 -0700245 remove_migration_pte(vma, old, new);
Christoph Lameter06972122006-06-23 02:03:35 -0700246
247 spin_unlock(&anon_vma->lock);
248}
249
250/*
Christoph Lameter04e62a22006-06-23 02:03:38 -0700251 * Get rid of all migration entries and replace them by
252 * references to the indicated page.
253 */
254static void remove_migration_ptes(struct page *old, struct page *new)
255{
256 if (PageAnon(new))
257 remove_anon_migration_ptes(old, new);
258 else
259 remove_file_migration_ptes(old, new);
260}
261
262/*
Christoph Lameter06972122006-06-23 02:03:35 -0700263 * Something used the pte of a page under migration. We need to
264 * get to the page and wait until migration is finished.
265 * When we return from this function the fault will be retried.
266 *
267 * This function is called from do_swap_page().
268 */
269void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
270 unsigned long address)
271{
272 pte_t *ptep, pte;
273 spinlock_t *ptl;
274 swp_entry_t entry;
275 struct page *page;
276
277 ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
278 pte = *ptep;
279 if (!is_swap_pte(pte))
280 goto out;
281
282 entry = pte_to_swp_entry(pte);
283 if (!is_migration_entry(entry))
284 goto out;
285
286 page = migration_entry_to_page(entry);
287
Nick Piggine2867812008-07-25 19:45:30 -0700288 /*
289 * Once radix-tree replacement of page migration started, page_count
290 * *must* be zero. And, we don't want to call wait_on_page_locked()
291 * against a page without get_page().
292 * So, we use get_page_unless_zero(), here. Even failed, page fault
293 * will occur again.
294 */
295 if (!get_page_unless_zero(page))
296 goto out;
Christoph Lameter06972122006-06-23 02:03:35 -0700297 pte_unmap_unlock(ptep, ptl);
298 wait_on_page_locked(page);
299 put_page(page);
300 return;
301out:
302 pte_unmap_unlock(ptep, ptl);
303}
304
Christoph Lameterb20a3502006-03-22 00:09:12 -0800305/*
Christoph Lameterc3fcf8a2006-06-23 02:03:32 -0700306 * Replace the page in the mapping.
Christoph Lameter5b5c7122006-06-23 02:03:29 -0700307 *
308 * The number of remaining references must be:
309 * 1 for anonymous pages without a mapping
310 * 2 for pages with a mapping
311 * 3 for pages with a mapping and PagePrivate set.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800312 */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700313static int migrate_page_move_mapping(struct address_space *mapping,
314 struct page *newpage, struct page *page)
Christoph Lameterb20a3502006-03-22 00:09:12 -0800315{
Nick Piggine2867812008-07-25 19:45:30 -0700316 int expected_count;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800317 void **pslot;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800318
Christoph Lameter6c5240a2006-06-23 02:03:37 -0700319 if (!mapping) {
Christoph Lameter0e8c7d02007-04-23 14:41:09 -0700320 /* Anonymous page without mapping */
Christoph Lameter6c5240a2006-06-23 02:03:37 -0700321 if (page_count(page) != 1)
322 return -EAGAIN;
323 return 0;
324 }
325
Nick Piggin19fd6232008-07-25 19:45:32 -0700326 spin_lock_irq(&mapping->tree_lock);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800327
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800328 pslot = radix_tree_lookup_slot(&mapping->page_tree,
329 page_index(page));
Christoph Lameterb20a3502006-03-22 00:09:12 -0800330
Nick Piggine2867812008-07-25 19:45:30 -0700331 expected_count = 2 + !!PagePrivate(page);
332 if (page_count(page) != expected_count ||
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800333 (struct page *)radix_tree_deref_slot(pslot) != page) {
Nick Piggin19fd6232008-07-25 19:45:32 -0700334 spin_unlock_irq(&mapping->tree_lock);
Christoph Lametere23ca002006-04-10 22:52:57 -0700335 return -EAGAIN;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800336 }
337
Nick Piggine2867812008-07-25 19:45:30 -0700338 if (!page_freeze_refs(page, expected_count)) {
Nick Piggin19fd6232008-07-25 19:45:32 -0700339 spin_unlock_irq(&mapping->tree_lock);
Nick Piggine2867812008-07-25 19:45:30 -0700340 return -EAGAIN;
341 }
342
Christoph Lameterb20a3502006-03-22 00:09:12 -0800343 /*
344 * Now we know that no one else is looking at the page.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800345 */
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800346 get_page(newpage); /* add cache reference */
Christoph Lameter6c5240a2006-06-23 02:03:37 -0700347#ifdef CONFIG_SWAP
Christoph Lameterb20a3502006-03-22 00:09:12 -0800348 if (PageSwapCache(page)) {
349 SetPageSwapCache(newpage);
350 set_page_private(newpage, page_private(page));
351 }
Christoph Lameter6c5240a2006-06-23 02:03:37 -0700352#endif
Christoph Lameterb20a3502006-03-22 00:09:12 -0800353
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800354 radix_tree_replace_slot(pslot, newpage);
355
Nick Piggine2867812008-07-25 19:45:30 -0700356 page_unfreeze_refs(page, expected_count);
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800357 /*
358 * Drop cache reference from old page.
359 * We know this isn't the last reference.
360 */
Christoph Lameterb20a3502006-03-22 00:09:12 -0800361 __put_page(page);
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800362
Christoph Lameter0e8c7d02007-04-23 14:41:09 -0700363 /*
364 * If moved to a different zone then also account
365 * the page for that zone. Other VM counters will be
366 * taken care of when we establish references to the
367 * new page and drop references to the old page.
368 *
369 * Note that anonymous pages are accounted for
370 * via NR_FILE_PAGES and NR_ANON_PAGES if they
371 * are mapped to swap space.
372 */
373 __dec_zone_page_state(page, NR_FILE_PAGES);
374 __inc_zone_page_state(newpage, NR_FILE_PAGES);
375
Nick Piggin19fd6232008-07-25 19:45:32 -0700376 spin_unlock_irq(&mapping->tree_lock);
377 if (!PageSwapCache(newpage))
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700378 mem_cgroup_uncharge_cache_page(page);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800379
380 return 0;
381}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800382
383/*
384 * Copy the page to its new location
385 */
Christoph Lametere7340f72006-06-23 02:03:29 -0700386static void migrate_page_copy(struct page *newpage, struct page *page)
Christoph Lameterb20a3502006-03-22 00:09:12 -0800387{
388 copy_highpage(newpage, page);
389
390 if (PageError(page))
391 SetPageError(newpage);
392 if (PageReferenced(page))
393 SetPageReferenced(newpage);
394 if (PageUptodate(page))
395 SetPageUptodate(newpage);
396 if (PageActive(page))
397 SetPageActive(newpage);
398 if (PageChecked(page))
399 SetPageChecked(newpage);
400 if (PageMappedToDisk(page))
401 SetPageMappedToDisk(newpage);
402
403 if (PageDirty(page)) {
404 clear_page_dirty_for_io(page);
Nick Piggin3a902c52008-04-30 00:55:16 -0700405 /*
406 * Want to mark the page and the radix tree as dirty, and
407 * redo the accounting that clear_page_dirty_for_io undid,
408 * but we can't use set_page_dirty because that function
409 * is actually a signal that all of the page has become dirty.
410 * Wheras only part of our page may be dirty.
411 */
412 __set_page_dirty_nobuffers(newpage);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800413 }
414
Christoph Lameter6c5240a2006-06-23 02:03:37 -0700415#ifdef CONFIG_SWAP
Christoph Lameterb20a3502006-03-22 00:09:12 -0800416 ClearPageSwapCache(page);
Christoph Lameter6c5240a2006-06-23 02:03:37 -0700417#endif
Christoph Lameterb20a3502006-03-22 00:09:12 -0800418 ClearPageActive(page);
419 ClearPagePrivate(page);
420 set_page_private(page, 0);
421 page->mapping = NULL;
422
423 /*
424 * If any waiters have accumulated on the new page then
425 * wake them up.
426 */
427 if (PageWriteback(newpage))
428 end_page_writeback(newpage);
429}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800430
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700431/************************************************************
432 * Migration functions
433 ***********************************************************/
434
435/* Always fail migration. Used for mappings that are not movable */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700436int fail_migrate_page(struct address_space *mapping,
437 struct page *newpage, struct page *page)
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700438{
439 return -EIO;
440}
441EXPORT_SYMBOL(fail_migrate_page);
442
Christoph Lameterb20a3502006-03-22 00:09:12 -0800443/*
444 * Common logic to directly migrate a single page suitable for
445 * pages that do not use PagePrivate.
446 *
447 * Pages are locked upon entry and exit.
448 */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700449int migrate_page(struct address_space *mapping,
450 struct page *newpage, struct page *page)
Christoph Lameterb20a3502006-03-22 00:09:12 -0800451{
452 int rc;
453
454 BUG_ON(PageWriteback(page)); /* Writeback must be complete */
455
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700456 rc = migrate_page_move_mapping(mapping, newpage, page);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800457
458 if (rc)
459 return rc;
460
461 migrate_page_copy(newpage, page);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800462 return 0;
463}
464EXPORT_SYMBOL(migrate_page);
465
David Howells93614012006-09-30 20:45:40 +0200466#ifdef CONFIG_BLOCK
Christoph Lameterb20a3502006-03-22 00:09:12 -0800467/*
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700468 * Migration function for pages with buffers. This function can only be used
469 * if the underlying filesystem guarantees that no other references to "page"
470 * exist.
471 */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700472int buffer_migrate_page(struct address_space *mapping,
473 struct page *newpage, struct page *page)
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700474{
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700475 struct buffer_head *bh, *head;
476 int rc;
477
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700478 if (!page_has_buffers(page))
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700479 return migrate_page(mapping, newpage, page);
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700480
481 head = page_buffers(page);
482
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700483 rc = migrate_page_move_mapping(mapping, newpage, page);
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700484
485 if (rc)
486 return rc;
487
488 bh = head;
489 do {
490 get_bh(bh);
491 lock_buffer(bh);
492 bh = bh->b_this_page;
493
494 } while (bh != head);
495
496 ClearPagePrivate(page);
497 set_page_private(newpage, page_private(page));
498 set_page_private(page, 0);
499 put_page(page);
500 get_page(newpage);
501
502 bh = head;
503 do {
504 set_bh_page(bh, newpage, bh_offset(bh));
505 bh = bh->b_this_page;
506
507 } while (bh != head);
508
509 SetPagePrivate(newpage);
510
511 migrate_page_copy(newpage, page);
512
513 bh = head;
514 do {
515 unlock_buffer(bh);
516 put_bh(bh);
517 bh = bh->b_this_page;
518
519 } while (bh != head);
520
521 return 0;
522}
523EXPORT_SYMBOL(buffer_migrate_page);
David Howells93614012006-09-30 20:45:40 +0200524#endif
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700525
Christoph Lameter04e62a22006-06-23 02:03:38 -0700526/*
527 * Writeback a page to clean the dirty state
528 */
529static int writeout(struct address_space *mapping, struct page *page)
530{
531 struct writeback_control wbc = {
532 .sync_mode = WB_SYNC_NONE,
533 .nr_to_write = 1,
534 .range_start = 0,
535 .range_end = LLONG_MAX,
536 .nonblocking = 1,
537 .for_reclaim = 1
538 };
539 int rc;
540
541 if (!mapping->a_ops->writepage)
542 /* No write method for the address space */
543 return -EINVAL;
544
545 if (!clear_page_dirty_for_io(page))
546 /* Someone else already triggered a write */
547 return -EAGAIN;
548
549 /*
550 * A dirty page may imply that the underlying filesystem has
551 * the page on some queue. So the page must be clean for
552 * migration. Writeout may mean we loose the lock and the
553 * page state is no longer what we checked for earlier.
554 * At this point we know that the migration attempt cannot
555 * be successful.
556 */
557 remove_migration_ptes(page, page);
558
559 rc = mapping->a_ops->writepage(page, &wbc);
560 if (rc < 0)
561 /* I/O Error writing */
562 return -EIO;
563
564 if (rc != AOP_WRITEPAGE_ACTIVATE)
565 /* unlocked. Relock */
566 lock_page(page);
567
568 return -EAGAIN;
569}
570
571/*
572 * Default handling if a filesystem does not provide a migration function.
573 */
Christoph Lameter8351a6e2006-06-23 02:03:33 -0700574static int fallback_migrate_page(struct address_space *mapping,
575 struct page *newpage, struct page *page)
576{
Christoph Lameter04e62a22006-06-23 02:03:38 -0700577 if (PageDirty(page))
578 return writeout(mapping, page);
Christoph Lameter8351a6e2006-06-23 02:03:33 -0700579
580 /*
581 * Buffers may be managed in a filesystem specific way.
582 * We must have no buffers or drop them.
583 */
David Howellsb398f6b2006-08-29 19:05:58 +0100584 if (PagePrivate(page) &&
Christoph Lameter8351a6e2006-06-23 02:03:33 -0700585 !try_to_release_page(page, GFP_KERNEL))
586 return -EAGAIN;
587
588 return migrate_page(mapping, newpage, page);
589}
590
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700591/*
Christoph Lametere24f0b82006-06-23 02:03:51 -0700592 * Move a page to a newly allocated page
593 * The page is locked and all ptes have been successfully removed.
594 *
595 * The new page will have replaced the old page if this function
596 * is successful.
597 */
598static int move_to_new_page(struct page *newpage, struct page *page)
599{
600 struct address_space *mapping;
601 int rc;
602
603 /*
604 * Block others from accessing the page when we get around to
605 * establishing additional references. We are the only one
606 * holding a reference to the new page at this point.
607 */
608 if (TestSetPageLocked(newpage))
609 BUG();
610
611 /* Prepare mapping for the new page.*/
612 newpage->index = page->index;
613 newpage->mapping = page->mapping;
614
615 mapping = page_mapping(page);
616 if (!mapping)
617 rc = migrate_page(mapping, newpage, page);
618 else if (mapping->a_ops->migratepage)
619 /*
620 * Most pages have a mapping and most filesystems
621 * should provide a migration function. Anonymous
622 * pages are part of swap space which also has its
623 * own migration function. This is the most common
624 * path for page migration.
625 */
626 rc = mapping->a_ops->migratepage(mapping,
627 newpage, page);
628 else
629 rc = fallback_migrate_page(mapping, newpage, page);
630
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -0800631 if (!rc) {
Christoph Lametere24f0b82006-06-23 02:03:51 -0700632 remove_migration_ptes(page, newpage);
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -0800633 } else
Christoph Lametere24f0b82006-06-23 02:03:51 -0700634 newpage->mapping = NULL;
635
636 unlock_page(newpage);
637
638 return rc;
639}
640
641/*
642 * Obtain the lock on page, remove all ptes and migrate the page
643 * to the newly allocated page in newpage.
644 */
Christoph Lameter95a402c2006-06-23 02:03:53 -0700645static int unmap_and_move(new_page_t get_new_page, unsigned long private,
646 struct page *page, int force)
Christoph Lametere24f0b82006-06-23 02:03:51 -0700647{
648 int rc = 0;
Christoph Lameter742755a2006-06-23 02:03:55 -0700649 int *result = NULL;
650 struct page *newpage = get_new_page(page, private, &result);
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700651 int rcu_locked = 0;
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -0800652 int charge = 0;
Christoph Lameter95a402c2006-06-23 02:03:53 -0700653
654 if (!newpage)
655 return -ENOMEM;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700656
657 if (page_count(page) == 1)
658 /* page was freed from under us. So we are done. */
Christoph Lameter95a402c2006-06-23 02:03:53 -0700659 goto move_newpage;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700660
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700661 charge = mem_cgroup_prepare_migration(page, newpage);
662 if (charge == -ENOMEM) {
663 rc = -ENOMEM;
664 goto move_newpage;
665 }
666 /* prepare cgroup just returns 0 or -ENOMEM */
667 BUG_ON(charge);
668
Christoph Lametere24f0b82006-06-23 02:03:51 -0700669 rc = -EAGAIN;
670 if (TestSetPageLocked(page)) {
671 if (!force)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700672 goto move_newpage;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700673 lock_page(page);
674 }
675
676 if (PageWriteback(page)) {
677 if (!force)
678 goto unlock;
679 wait_on_page_writeback(page);
680 }
Christoph Lametere24f0b82006-06-23 02:03:51 -0700681 /*
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700682 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
683 * we cannot notice that anon_vma is freed while we migrates a page.
684 * This rcu_read_lock() delays freeing anon_vma pointer until the end
685 * of migration. File cache pages are no problem because of page_lock()
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700686 * File Caches may use write_page() or lock_page() in migration, then,
687 * just care Anon page here.
Christoph Lametere24f0b82006-06-23 02:03:51 -0700688 */
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700689 if (PageAnon(page)) {
690 rcu_read_lock();
691 rcu_locked = 1;
692 }
Shaohua Li62e1c552008-02-04 22:29:33 -0800693
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700694 /*
Shaohua Li62e1c552008-02-04 22:29:33 -0800695 * Corner case handling:
696 * 1. When a new swap-cache page is read into, it is added to the LRU
697 * and treated as swapcache but it has no rmap yet.
698 * Calling try_to_unmap() against a page->mapping==NULL page will
699 * trigger a BUG. So handle it here.
700 * 2. An orphaned page (see truncate_complete_page) might have
701 * fs-private metadata. The page can be picked up due to memory
702 * offlining. Everywhere else except page reclaim, the page is
703 * invisible to the vm, so the page can not be migrated. So try to
704 * free the metadata, so the page can be freed.
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700705 */
Shaohua Li62e1c552008-02-04 22:29:33 -0800706 if (!page->mapping) {
707 if (!PageAnon(page) && PagePrivate(page)) {
708 /*
709 * Go direct to try_to_free_buffers() here because
710 * a) that's what try_to_release_page() would do anyway
711 * b) we may be under rcu_read_lock() here, so we can't
712 * use GFP_KERNEL which is what try_to_release_page()
713 * needs to be effective.
714 */
715 try_to_free_buffers(page);
716 }
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700717 goto rcu_unlock;
Shaohua Li62e1c552008-02-04 22:29:33 -0800718 }
719
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700720 /* Establish migration ptes or remove ptes */
Christoph Lametere6a15302006-06-25 05:46:49 -0700721 try_to_unmap(page, 1);
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700722
Christoph Lametere6a15302006-06-25 05:46:49 -0700723 if (!page_mapped(page))
724 rc = move_to_new_page(newpage, page);
Christoph Lametere24f0b82006-06-23 02:03:51 -0700725
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700726 if (rc)
Christoph Lametere24f0b82006-06-23 02:03:51 -0700727 remove_migration_ptes(page, page);
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700728rcu_unlock:
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700729 if (rcu_locked)
730 rcu_read_unlock();
Christoph Lametere6a15302006-06-25 05:46:49 -0700731
Christoph Lametere24f0b82006-06-23 02:03:51 -0700732unlock:
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700733
Christoph Lametere24f0b82006-06-23 02:03:51 -0700734 unlock_page(page);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700735
Christoph Lametere24f0b82006-06-23 02:03:51 -0700736 if (rc != -EAGAIN) {
Christoph Lameteraaa994b2006-06-23 02:03:52 -0700737 /*
738 * A page that has been migrated has all references
739 * removed and will be freed. A page that has not been
740 * migrated will have kepts its references and be
741 * restored.
742 */
743 list_del(&page->lru);
744 move_to_lru(page);
Christoph Lametere24f0b82006-06-23 02:03:51 -0700745 }
Christoph Lameter95a402c2006-06-23 02:03:53 -0700746
747move_newpage:
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700748 if (!charge)
749 mem_cgroup_end_migration(newpage);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700750 /*
751 * Move the new page to the LRU. If migration was not successful
752 * then this will free the page.
753 */
754 move_to_lru(newpage);
Christoph Lameter742755a2006-06-23 02:03:55 -0700755 if (result) {
756 if (rc)
757 *result = rc;
758 else
759 *result = page_to_nid(newpage);
760 }
Christoph Lametere24f0b82006-06-23 02:03:51 -0700761 return rc;
762}
763
764/*
Christoph Lameterb20a3502006-03-22 00:09:12 -0800765 * migrate_pages
766 *
Christoph Lameter95a402c2006-06-23 02:03:53 -0700767 * The function takes one list of pages to migrate and a function
768 * that determines from the page to be migrated and the private data
769 * the target of the move and allocates the page.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800770 *
771 * The function returns after 10 attempts or if no pages
772 * are movable anymore because to has become empty
Christoph Lameteraaa994b2006-06-23 02:03:52 -0700773 * or no retryable pages exist anymore. All pages will be
Gabriel Craciunescue9534b32007-10-20 02:13:26 +0200774 * returned to the LRU or freed.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800775 *
Christoph Lameter95a402c2006-06-23 02:03:53 -0700776 * Return: Number of pages not migrated or error code.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800777 */
Christoph Lameter95a402c2006-06-23 02:03:53 -0700778int migrate_pages(struct list_head *from,
779 new_page_t get_new_page, unsigned long private)
Christoph Lameterb20a3502006-03-22 00:09:12 -0800780{
Christoph Lametere24f0b82006-06-23 02:03:51 -0700781 int retry = 1;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800782 int nr_failed = 0;
783 int pass = 0;
784 struct page *page;
785 struct page *page2;
786 int swapwrite = current->flags & PF_SWAPWRITE;
787 int rc;
788
789 if (!swapwrite)
790 current->flags |= PF_SWAPWRITE;
791
Christoph Lametere24f0b82006-06-23 02:03:51 -0700792 for(pass = 0; pass < 10 && retry; pass++) {
793 retry = 0;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800794
Christoph Lametere24f0b82006-06-23 02:03:51 -0700795 list_for_each_entry_safe(page, page2, from, lru) {
Christoph Lametere24f0b82006-06-23 02:03:51 -0700796 cond_resched();
Christoph Lameterb20a3502006-03-22 00:09:12 -0800797
Christoph Lameter95a402c2006-06-23 02:03:53 -0700798 rc = unmap_and_move(get_new_page, private,
799 page, pass > 2);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800800
Christoph Lametere24f0b82006-06-23 02:03:51 -0700801 switch(rc) {
Christoph Lameter95a402c2006-06-23 02:03:53 -0700802 case -ENOMEM:
803 goto out;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700804 case -EAGAIN:
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700805 retry++;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700806 break;
807 case 0:
Christoph Lametere24f0b82006-06-23 02:03:51 -0700808 break;
809 default:
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700810 /* Permanent failure */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700811 nr_failed++;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700812 break;
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700813 }
Christoph Lameterb20a3502006-03-22 00:09:12 -0800814 }
815 }
Christoph Lameter95a402c2006-06-23 02:03:53 -0700816 rc = 0;
817out:
Christoph Lameterb20a3502006-03-22 00:09:12 -0800818 if (!swapwrite)
819 current->flags &= ~PF_SWAPWRITE;
820
Christoph Lameteraaa994b2006-06-23 02:03:52 -0700821 putback_lru_pages(from);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700822
823 if (rc)
824 return rc;
825
Christoph Lameterb20a3502006-03-22 00:09:12 -0800826 return nr_failed + retry;
827}
828
Christoph Lameter742755a2006-06-23 02:03:55 -0700829#ifdef CONFIG_NUMA
830/*
831 * Move a list of individual pages
832 */
833struct page_to_node {
834 unsigned long addr;
835 struct page *page;
836 int node;
837 int status;
838};
839
840static struct page *new_page_node(struct page *p, unsigned long private,
841 int **result)
842{
843 struct page_to_node *pm = (struct page_to_node *)private;
844
845 while (pm->node != MAX_NUMNODES && pm->page != p)
846 pm++;
847
848 if (pm->node == MAX_NUMNODES)
849 return NULL;
850
851 *result = &pm->status;
852
Mel Gorman769848c2007-07-17 04:03:05 -0700853 return alloc_pages_node(pm->node,
854 GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0);
Christoph Lameter742755a2006-06-23 02:03:55 -0700855}
856
857/*
858 * Move a set of pages as indicated in the pm array. The addr
859 * field must be set to the virtual address of the page to be moved
860 * and the node number must contain a valid target node.
861 */
862static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm,
863 int migrate_all)
864{
865 int err;
866 struct page_to_node *pp;
867 LIST_HEAD(pagelist);
868
869 down_read(&mm->mmap_sem);
870
871 /*
872 * Build a list of pages to migrate
873 */
874 migrate_prep();
875 for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
876 struct vm_area_struct *vma;
877 struct page *page;
878
879 /*
880 * A valid page pointer that will not match any of the
881 * pages that will be moved.
882 */
883 pp->page = ZERO_PAGE(0);
884
885 err = -EFAULT;
886 vma = find_vma(mm, pp->addr);
Christoph Lameter0dc952d2007-03-05 00:30:33 -0800887 if (!vma || !vma_migratable(vma))
Christoph Lameter742755a2006-06-23 02:03:55 -0700888 goto set_status;
889
890 page = follow_page(vma, pp->addr, FOLL_GET);
Linus Torvalds89f5b7d2008-06-20 11:18:25 -0700891
892 err = PTR_ERR(page);
893 if (IS_ERR(page))
894 goto set_status;
895
Christoph Lameter742755a2006-06-23 02:03:55 -0700896 err = -ENOENT;
897 if (!page)
898 goto set_status;
899
900 if (PageReserved(page)) /* Check for zero page */
901 goto put_and_set;
902
903 pp->page = page;
904 err = page_to_nid(page);
905
906 if (err == pp->node)
907 /*
908 * Node already in the right place
909 */
910 goto put_and_set;
911
912 err = -EACCES;
913 if (page_mapcount(page) > 1 &&
914 !migrate_all)
915 goto put_and_set;
916
917 err = isolate_lru_page(page, &pagelist);
918put_and_set:
919 /*
920 * Either remove the duplicate refcount from
921 * isolate_lru_page() or drop the page ref if it was
922 * not isolated.
923 */
924 put_page(page);
925set_status:
926 pp->status = err;
927 }
928
929 if (!list_empty(&pagelist))
930 err = migrate_pages(&pagelist, new_page_node,
931 (unsigned long)pm);
932 else
933 err = -ENOENT;
934
935 up_read(&mm->mmap_sem);
936 return err;
937}
938
939/*
940 * Determine the nodes of a list of pages. The addr in the pm array
941 * must have been set to the virtual address of which we want to determine
942 * the node number.
943 */
944static int do_pages_stat(struct mm_struct *mm, struct page_to_node *pm)
945{
946 down_read(&mm->mmap_sem);
947
948 for ( ; pm->node != MAX_NUMNODES; pm++) {
949 struct vm_area_struct *vma;
950 struct page *page;
951 int err;
952
953 err = -EFAULT;
954 vma = find_vma(mm, pm->addr);
955 if (!vma)
956 goto set_status;
957
958 page = follow_page(vma, pm->addr, 0);
Linus Torvalds89f5b7d2008-06-20 11:18:25 -0700959
960 err = PTR_ERR(page);
961 if (IS_ERR(page))
962 goto set_status;
963
Christoph Lameter742755a2006-06-23 02:03:55 -0700964 err = -ENOENT;
965 /* Use PageReserved to check for zero page */
966 if (!page || PageReserved(page))
967 goto set_status;
968
969 err = page_to_nid(page);
970set_status:
971 pm->status = err;
972 }
973
974 up_read(&mm->mmap_sem);
975 return 0;
976}
977
978/*
979 * Move a list of pages in the address space of the currently executing
980 * process.
981 */
982asmlinkage long sys_move_pages(pid_t pid, unsigned long nr_pages,
983 const void __user * __user *pages,
984 const int __user *nodes,
985 int __user *status, int flags)
986{
987 int err = 0;
988 int i;
989 struct task_struct *task;
990 nodemask_t task_nodes;
991 struct mm_struct *mm;
992 struct page_to_node *pm = NULL;
993
994 /* Check flags */
995 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
996 return -EINVAL;
997
998 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
999 return -EPERM;
1000
1001 /* Find the mm_struct */
1002 read_lock(&tasklist_lock);
Pavel Emelyanov228ebcb2007-10-18 23:40:16 -07001003 task = pid ? find_task_by_vpid(pid) : current;
Christoph Lameter742755a2006-06-23 02:03:55 -07001004 if (!task) {
1005 read_unlock(&tasklist_lock);
1006 return -ESRCH;
1007 }
1008 mm = get_task_mm(task);
1009 read_unlock(&tasklist_lock);
1010
1011 if (!mm)
1012 return -EINVAL;
1013
1014 /*
1015 * Check if this process has the right to modify the specified
1016 * process. The right exists if the process has administrative
1017 * capabilities, superuser privileges or the same
1018 * userid as the target process.
1019 */
1020 if ((current->euid != task->suid) && (current->euid != task->uid) &&
1021 (current->uid != task->suid) && (current->uid != task->uid) &&
1022 !capable(CAP_SYS_NICE)) {
1023 err = -EPERM;
1024 goto out2;
1025 }
1026
David Quigley86c3a762006-06-23 02:04:02 -07001027 err = security_task_movememory(task);
1028 if (err)
1029 goto out2;
1030
1031
Christoph Lameter742755a2006-06-23 02:03:55 -07001032 task_nodes = cpuset_mems_allowed(task);
1033
1034 /* Limit nr_pages so that the multiplication may not overflow */
1035 if (nr_pages >= ULONG_MAX / sizeof(struct page_to_node) - 1) {
1036 err = -E2BIG;
1037 goto out2;
1038 }
1039
1040 pm = vmalloc((nr_pages + 1) * sizeof(struct page_to_node));
1041 if (!pm) {
1042 err = -ENOMEM;
1043 goto out2;
1044 }
1045
1046 /*
1047 * Get parameters from user space and initialize the pm
1048 * array. Return various errors if the user did something wrong.
1049 */
1050 for (i = 0; i < nr_pages; i++) {
Al Viro9d966d42007-10-14 19:34:10 +01001051 const void __user *p;
Christoph Lameter742755a2006-06-23 02:03:55 -07001052
1053 err = -EFAULT;
1054 if (get_user(p, pages + i))
1055 goto out;
1056
1057 pm[i].addr = (unsigned long)p;
1058 if (nodes) {
1059 int node;
1060
1061 if (get_user(node, nodes + i))
1062 goto out;
1063
1064 err = -ENODEV;
Christoph Lameter56bbd652007-10-16 01:25:35 -07001065 if (!node_state(node, N_HIGH_MEMORY))
Christoph Lameter742755a2006-06-23 02:03:55 -07001066 goto out;
1067
1068 err = -EACCES;
1069 if (!node_isset(node, task_nodes))
1070 goto out;
1071
1072 pm[i].node = node;
Stephen Rothwell8ce08462006-11-02 22:07:28 -08001073 } else
1074 pm[i].node = 0; /* anything to not match MAX_NUMNODES */
Christoph Lameter742755a2006-06-23 02:03:55 -07001075 }
1076 /* End marker */
1077 pm[nr_pages].node = MAX_NUMNODES;
1078
1079 if (nodes)
1080 err = do_move_pages(mm, pm, flags & MPOL_MF_MOVE_ALL);
1081 else
1082 err = do_pages_stat(mm, pm);
1083
1084 if (err >= 0)
1085 /* Return status information */
1086 for (i = 0; i < nr_pages; i++)
1087 if (put_user(pm[i].status, status + i))
1088 err = -EFAULT;
1089
1090out:
1091 vfree(pm);
1092out2:
1093 mmput(mm);
1094 return err;
1095}
Christoph Lameter742755a2006-06-23 02:03:55 -07001096
Christoph Lameter7b2259b2006-06-25 05:46:48 -07001097/*
1098 * Call migration functions in the vma_ops that may prepare
1099 * memory in a vm for migration. migration functions may perform
1100 * the migration for vmas that do not have an underlying page struct.
1101 */
1102int migrate_vmas(struct mm_struct *mm, const nodemask_t *to,
1103 const nodemask_t *from, unsigned long flags)
1104{
1105 struct vm_area_struct *vma;
1106 int err = 0;
1107
1108 for(vma = mm->mmap; vma->vm_next && !err; vma = vma->vm_next) {
1109 if (vma->vm_ops && vma->vm_ops->migrate) {
1110 err = vma->vm_ops->migrate(vma, to, from, flags);
1111 if (err)
1112 break;
1113 }
1114 }
1115 return err;
1116}
Gerald Schaefer83d16742008-07-23 21:28:22 -07001117#endif