blob: 690d0de993afb996997e5c4bca13a85d10ec0c0b [file] [log] [blame]
Christoph Lameterb20a3502006-03-22 00:09:12 -08001/*
2 * Memory Migration functionality - linux/mm/migration.c
3 *
4 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
5 *
6 * Page migration was first developed in the context of the memory hotplug
7 * project. The main authors of the migration code are:
8 *
9 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
10 * Hirokazu Takahashi <taka@valinux.co.jp>
11 * Dave Hansen <haveblue@us.ibm.com>
Christoph Lametercde53532008-07-04 09:59:22 -070012 * Christoph Lameter
Christoph Lameterb20a3502006-03-22 00:09:12 -080013 */
14
15#include <linux/migrate.h>
16#include <linux/module.h>
17#include <linux/swap.h>
Christoph Lameter06972122006-06-23 02:03:35 -070018#include <linux/swapops.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080019#include <linux/pagemap.h>
Christoph Lametere23ca002006-04-10 22:52:57 -070020#include <linux/buffer_head.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080021#include <linux/mm_inline.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070022#include <linux/nsproxy.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080023#include <linux/pagevec.h>
Hugh Dickinse9995ef2009-12-14 17:59:31 -080024#include <linux/ksm.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080025#include <linux/rmap.h>
26#include <linux/topology.h>
27#include <linux/cpu.h>
28#include <linux/cpuset.h>
Christoph Lameter04e62a22006-06-23 02:03:38 -070029#include <linux/writeback.h>
Christoph Lameter742755a2006-06-23 02:03:55 -070030#include <linux/mempolicy.h>
31#include <linux/vmalloc.h>
David Quigley86c3a762006-06-23 02:04:02 -070032#include <linux/security.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080033#include <linux/memcontrol.h>
Adrian Bunk4f5ca262008-07-23 21:27:02 -070034#include <linux/syscalls.h>
Naoya Horiguchi290408d2010-09-08 10:19:35 +090035#include <linux/hugetlb.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090036#include <linux/gfp.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080037
Michal Nazarewicz0d1836c2010-12-21 17:24:26 -080038#include <asm/tlbflush.h>
39
Christoph Lameterb20a3502006-03-22 00:09:12 -080040#include "internal.h"
41
Christoph Lameterb20a3502006-03-22 00:09:12 -080042#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
43
44/*
Christoph Lameter742755a2006-06-23 02:03:55 -070045 * migrate_prep() needs to be called before we start compiling a list of pages
Mel Gorman748446b2010-05-24 14:32:27 -070046 * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
47 * undesirable, use migrate_prep_local()
Christoph Lameterb20a3502006-03-22 00:09:12 -080048 */
49int migrate_prep(void)
50{
Christoph Lameterb20a3502006-03-22 00:09:12 -080051 /*
52 * Clear the LRU lists so pages can be isolated.
53 * Note that pages may be moved off the LRU after we have
54 * drained them. Those pages will fail to migrate like other
55 * pages that may be busy.
56 */
57 lru_add_drain_all();
58
59 return 0;
60}
61
Mel Gorman748446b2010-05-24 14:32:27 -070062/* Do the necessary work of migrate_prep but not if it involves other CPUs */
63int migrate_prep_local(void)
64{
65 lru_add_drain();
66
67 return 0;
68}
69
Christoph Lameterb20a3502006-03-22 00:09:12 -080070/*
Lee Schermerhorn894bc312008-10-18 20:26:39 -070071 * Add isolated pages on the list back to the LRU under page lock
72 * to avoid leaking evictable pages back onto unevictable list.
Christoph Lameterb20a3502006-03-22 00:09:12 -080073 */
Minchan Kime13861d2010-05-24 14:31:59 -070074void putback_lru_pages(struct list_head *l)
Christoph Lameterb20a3502006-03-22 00:09:12 -080075{
76 struct page *page;
77 struct page *page2;
Christoph Lameterb20a3502006-03-22 00:09:12 -080078
79 list_for_each_entry_safe(page, page2, l, lru) {
Christoph Lametere24f0b82006-06-23 02:03:51 -070080 list_del(&page->lru);
KOSAKI Motohiroa7312862009-09-21 17:01:37 -070081 dec_zone_page_state(page, NR_ISOLATED_ANON +
Johannes Weiner6c0b1352009-09-21 17:02:59 -070082 page_is_file_cache(page));
Lee Schermerhorn894bc312008-10-18 20:26:39 -070083 putback_lru_page(page);
Christoph Lameterb20a3502006-03-22 00:09:12 -080084 }
Christoph Lameterb20a3502006-03-22 00:09:12 -080085}
86
Christoph Lameter06972122006-06-23 02:03:35 -070087/*
88 * Restore a potential migration pte to a working pte entry
89 */
Hugh Dickinse9995ef2009-12-14 17:59:31 -080090static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
91 unsigned long addr, void *old)
Christoph Lameter06972122006-06-23 02:03:35 -070092{
93 struct mm_struct *mm = vma->vm_mm;
94 swp_entry_t entry;
95 pgd_t *pgd;
96 pud_t *pud;
97 pmd_t *pmd;
98 pte_t *ptep, pte;
99 spinlock_t *ptl;
100
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900101 if (unlikely(PageHuge(new))) {
102 ptep = huge_pte_offset(mm, addr);
103 if (!ptep)
104 goto out;
105 ptl = &mm->page_table_lock;
106 } else {
107 pgd = pgd_offset(mm, addr);
108 if (!pgd_present(*pgd))
109 goto out;
Christoph Lameter06972122006-06-23 02:03:35 -0700110
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900111 pud = pud_offset(pgd, addr);
112 if (!pud_present(*pud))
113 goto out;
Christoph Lameter06972122006-06-23 02:03:35 -0700114
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900115 pmd = pmd_offset(pud, addr);
116 if (!pmd_present(*pmd))
117 goto out;
Christoph Lameter06972122006-06-23 02:03:35 -0700118
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900119 ptep = pte_offset_map(pmd, addr);
Christoph Lameter06972122006-06-23 02:03:35 -0700120
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900121 if (!is_swap_pte(*ptep)) {
122 pte_unmap(ptep);
123 goto out;
124 }
Christoph Lameter06972122006-06-23 02:03:35 -0700125
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900126 ptl = pte_lockptr(mm, pmd);
127 }
128
Christoph Lameter06972122006-06-23 02:03:35 -0700129 spin_lock(ptl);
130 pte = *ptep;
131 if (!is_swap_pte(pte))
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800132 goto unlock;
Christoph Lameter06972122006-06-23 02:03:35 -0700133
134 entry = pte_to_swp_entry(pte);
135
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800136 if (!is_migration_entry(entry) ||
137 migration_entry_to_page(entry) != old)
138 goto unlock;
Christoph Lameter06972122006-06-23 02:03:35 -0700139
Christoph Lameter06972122006-06-23 02:03:35 -0700140 get_page(new);
141 pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
142 if (is_write_migration_entry(entry))
143 pte = pte_mkwrite(pte);
Andi Kleen3ef8fd72010-10-11 16:03:21 +0200144#ifdef CONFIG_HUGETLB_PAGE
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900145 if (PageHuge(new))
146 pte = pte_mkhuge(pte);
Andi Kleen3ef8fd72010-10-11 16:03:21 +0200147#endif
KAMEZAWA Hiroyuki97ee0522007-10-16 01:25:43 -0700148 flush_cache_page(vma, addr, pte_pfn(pte));
Christoph Lameter06972122006-06-23 02:03:35 -0700149 set_pte_at(mm, addr, ptep, pte);
Christoph Lameter04e62a22006-06-23 02:03:38 -0700150
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900151 if (PageHuge(new)) {
152 if (PageAnon(new))
153 hugepage_add_anon_rmap(new, vma, addr);
154 else
155 page_dup_rmap(new);
156 } else if (PageAnon(new))
Christoph Lameter04e62a22006-06-23 02:03:38 -0700157 page_add_anon_rmap(new, vma, addr);
158 else
159 page_add_file_rmap(new);
160
161 /* No need to invalidate - it was non-present before */
Russell King4b3073e2009-12-18 16:40:18 +0000162 update_mmu_cache(vma, addr, ptep);
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800163unlock:
Christoph Lameter06972122006-06-23 02:03:35 -0700164 pte_unmap_unlock(ptep, ptl);
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800165out:
166 return SWAP_AGAIN;
Christoph Lameter06972122006-06-23 02:03:35 -0700167}
168
169/*
Christoph Lameter04e62a22006-06-23 02:03:38 -0700170 * Get rid of all migration entries and replace them by
171 * references to the indicated page.
172 */
173static void remove_migration_ptes(struct page *old, struct page *new)
174{
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800175 rmap_walk(new, remove_migration_pte, old);
Christoph Lameter04e62a22006-06-23 02:03:38 -0700176}
177
178/*
Christoph Lameter06972122006-06-23 02:03:35 -0700179 * Something used the pte of a page under migration. We need to
180 * get to the page and wait until migration is finished.
181 * When we return from this function the fault will be retried.
182 *
183 * This function is called from do_swap_page().
184 */
185void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
186 unsigned long address)
187{
188 pte_t *ptep, pte;
189 spinlock_t *ptl;
190 swp_entry_t entry;
191 struct page *page;
192
193 ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
194 pte = *ptep;
195 if (!is_swap_pte(pte))
196 goto out;
197
198 entry = pte_to_swp_entry(pte);
199 if (!is_migration_entry(entry))
200 goto out;
201
202 page = migration_entry_to_page(entry);
203
Nick Piggine2867812008-07-25 19:45:30 -0700204 /*
205 * Once radix-tree replacement of page migration started, page_count
206 * *must* be zero. And, we don't want to call wait_on_page_locked()
207 * against a page without get_page().
208 * So, we use get_page_unless_zero(), here. Even failed, page fault
209 * will occur again.
210 */
211 if (!get_page_unless_zero(page))
212 goto out;
Christoph Lameter06972122006-06-23 02:03:35 -0700213 pte_unmap_unlock(ptep, ptl);
214 wait_on_page_locked(page);
215 put_page(page);
216 return;
217out:
218 pte_unmap_unlock(ptep, ptl);
219}
220
Christoph Lameterb20a3502006-03-22 00:09:12 -0800221/*
Christoph Lameterc3fcf8a2006-06-23 02:03:32 -0700222 * Replace the page in the mapping.
Christoph Lameter5b5c7122006-06-23 02:03:29 -0700223 *
224 * The number of remaining references must be:
225 * 1 for anonymous pages without a mapping
226 * 2 for pages with a mapping
David Howells266cf652009-04-03 16:42:36 +0100227 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800228 */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700229static int migrate_page_move_mapping(struct address_space *mapping,
230 struct page *newpage, struct page *page)
Christoph Lameterb20a3502006-03-22 00:09:12 -0800231{
Nick Piggine2867812008-07-25 19:45:30 -0700232 int expected_count;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800233 void **pslot;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800234
Christoph Lameter6c5240a2006-06-23 02:03:37 -0700235 if (!mapping) {
Christoph Lameter0e8c7d02007-04-23 14:41:09 -0700236 /* Anonymous page without mapping */
Christoph Lameter6c5240a2006-06-23 02:03:37 -0700237 if (page_count(page) != 1)
238 return -EAGAIN;
239 return 0;
240 }
241
Nick Piggin19fd6232008-07-25 19:45:32 -0700242 spin_lock_irq(&mapping->tree_lock);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800243
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800244 pslot = radix_tree_lookup_slot(&mapping->page_tree,
245 page_index(page));
Christoph Lameterb20a3502006-03-22 00:09:12 -0800246
Johannes Weineredcf4742009-09-21 17:02:59 -0700247 expected_count = 2 + page_has_private(page);
Nick Piggine2867812008-07-25 19:45:30 -0700248 if (page_count(page) != expected_count ||
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800249 (struct page *)radix_tree_deref_slot(pslot) != page) {
Nick Piggin19fd6232008-07-25 19:45:32 -0700250 spin_unlock_irq(&mapping->tree_lock);
Christoph Lametere23ca002006-04-10 22:52:57 -0700251 return -EAGAIN;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800252 }
253
Nick Piggine2867812008-07-25 19:45:30 -0700254 if (!page_freeze_refs(page, expected_count)) {
Nick Piggin19fd6232008-07-25 19:45:32 -0700255 spin_unlock_irq(&mapping->tree_lock);
Nick Piggine2867812008-07-25 19:45:30 -0700256 return -EAGAIN;
257 }
258
Christoph Lameterb20a3502006-03-22 00:09:12 -0800259 /*
260 * Now we know that no one else is looking at the page.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800261 */
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800262 get_page(newpage); /* add cache reference */
Christoph Lameterb20a3502006-03-22 00:09:12 -0800263 if (PageSwapCache(page)) {
264 SetPageSwapCache(newpage);
265 set_page_private(newpage, page_private(page));
266 }
267
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800268 radix_tree_replace_slot(pslot, newpage);
269
Nick Piggine2867812008-07-25 19:45:30 -0700270 page_unfreeze_refs(page, expected_count);
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800271 /*
272 * Drop cache reference from old page.
273 * We know this isn't the last reference.
274 */
Christoph Lameterb20a3502006-03-22 00:09:12 -0800275 __put_page(page);
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800276
Christoph Lameter0e8c7d02007-04-23 14:41:09 -0700277 /*
278 * If moved to a different zone then also account
279 * the page for that zone. Other VM counters will be
280 * taken care of when we establish references to the
281 * new page and drop references to the old page.
282 *
283 * Note that anonymous pages are accounted for
284 * via NR_FILE_PAGES and NR_ANON_PAGES if they
285 * are mapped to swap space.
286 */
287 __dec_zone_page_state(page, NR_FILE_PAGES);
288 __inc_zone_page_state(newpage, NR_FILE_PAGES);
KOSAKI Motohiro4b021082009-09-21 17:01:33 -0700289 if (PageSwapBacked(page)) {
290 __dec_zone_page_state(page, NR_SHMEM);
291 __inc_zone_page_state(newpage, NR_SHMEM);
292 }
Nick Piggin19fd6232008-07-25 19:45:32 -0700293 spin_unlock_irq(&mapping->tree_lock);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800294
295 return 0;
296}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800297
298/*
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900299 * The expected number of remaining references is the same as that
300 * of migrate_page_move_mapping().
301 */
302int migrate_huge_page_move_mapping(struct address_space *mapping,
303 struct page *newpage, struct page *page)
304{
305 int expected_count;
306 void **pslot;
307
308 if (!mapping) {
309 if (page_count(page) != 1)
310 return -EAGAIN;
311 return 0;
312 }
313
314 spin_lock_irq(&mapping->tree_lock);
315
316 pslot = radix_tree_lookup_slot(&mapping->page_tree,
317 page_index(page));
318
319 expected_count = 2 + page_has_private(page);
320 if (page_count(page) != expected_count ||
321 (struct page *)radix_tree_deref_slot(pslot) != page) {
322 spin_unlock_irq(&mapping->tree_lock);
323 return -EAGAIN;
324 }
325
326 if (!page_freeze_refs(page, expected_count)) {
327 spin_unlock_irq(&mapping->tree_lock);
328 return -EAGAIN;
329 }
330
331 get_page(newpage);
332
333 radix_tree_replace_slot(pslot, newpage);
334
335 page_unfreeze_refs(page, expected_count);
336
337 __put_page(page);
338
339 spin_unlock_irq(&mapping->tree_lock);
340 return 0;
341}
342
343/*
Christoph Lameterb20a3502006-03-22 00:09:12 -0800344 * Copy the page to its new location
345 */
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900346void migrate_page_copy(struct page *newpage, struct page *page)
Christoph Lameterb20a3502006-03-22 00:09:12 -0800347{
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900348 if (PageHuge(page))
349 copy_huge_page(newpage, page);
350 else
351 copy_highpage(newpage, page);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800352
353 if (PageError(page))
354 SetPageError(newpage);
355 if (PageReferenced(page))
356 SetPageReferenced(newpage);
357 if (PageUptodate(page))
358 SetPageUptodate(newpage);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700359 if (TestClearPageActive(page)) {
360 VM_BUG_ON(PageUnevictable(page));
Christoph Lameterb20a3502006-03-22 00:09:12 -0800361 SetPageActive(newpage);
Lee Schermerhorn418b27e2009-12-14 17:59:54 -0800362 } else if (TestClearPageUnevictable(page))
363 SetPageUnevictable(newpage);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800364 if (PageChecked(page))
365 SetPageChecked(newpage);
366 if (PageMappedToDisk(page))
367 SetPageMappedToDisk(newpage);
368
369 if (PageDirty(page)) {
370 clear_page_dirty_for_io(page);
Nick Piggin3a902c52008-04-30 00:55:16 -0700371 /*
372 * Want to mark the page and the radix tree as dirty, and
373 * redo the accounting that clear_page_dirty_for_io undid,
374 * but we can't use set_page_dirty because that function
375 * is actually a signal that all of the page has become dirty.
376 * Wheras only part of our page may be dirty.
377 */
378 __set_page_dirty_nobuffers(newpage);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800379 }
380
Nick Pigginb291f002008-10-18 20:26:44 -0700381 mlock_migrate_page(newpage, page);
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800382 ksm_migrate_page(newpage, page);
Nick Pigginb291f002008-10-18 20:26:44 -0700383
Christoph Lameterb20a3502006-03-22 00:09:12 -0800384 ClearPageSwapCache(page);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800385 ClearPagePrivate(page);
386 set_page_private(page, 0);
387 page->mapping = NULL;
388
389 /*
390 * If any waiters have accumulated on the new page then
391 * wake them up.
392 */
393 if (PageWriteback(newpage))
394 end_page_writeback(newpage);
395}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800396
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700397/************************************************************
398 * Migration functions
399 ***********************************************************/
400
401/* Always fail migration. Used for mappings that are not movable */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700402int fail_migrate_page(struct address_space *mapping,
403 struct page *newpage, struct page *page)
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700404{
405 return -EIO;
406}
407EXPORT_SYMBOL(fail_migrate_page);
408
Christoph Lameterb20a3502006-03-22 00:09:12 -0800409/*
410 * Common logic to directly migrate a single page suitable for
David Howells266cf652009-04-03 16:42:36 +0100411 * pages that do not use PagePrivate/PagePrivate2.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800412 *
413 * Pages are locked upon entry and exit.
414 */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700415int migrate_page(struct address_space *mapping,
416 struct page *newpage, struct page *page)
Christoph Lameterb20a3502006-03-22 00:09:12 -0800417{
418 int rc;
419
420 BUG_ON(PageWriteback(page)); /* Writeback must be complete */
421
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700422 rc = migrate_page_move_mapping(mapping, newpage, page);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800423
424 if (rc)
425 return rc;
426
427 migrate_page_copy(newpage, page);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800428 return 0;
429}
430EXPORT_SYMBOL(migrate_page);
431
David Howells93614012006-09-30 20:45:40 +0200432#ifdef CONFIG_BLOCK
Christoph Lameterb20a3502006-03-22 00:09:12 -0800433/*
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700434 * Migration function for pages with buffers. This function can only be used
435 * if the underlying filesystem guarantees that no other references to "page"
436 * exist.
437 */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700438int buffer_migrate_page(struct address_space *mapping,
439 struct page *newpage, struct page *page)
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700440{
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700441 struct buffer_head *bh, *head;
442 int rc;
443
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700444 if (!page_has_buffers(page))
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700445 return migrate_page(mapping, newpage, page);
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700446
447 head = page_buffers(page);
448
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700449 rc = migrate_page_move_mapping(mapping, newpage, page);
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700450
451 if (rc)
452 return rc;
453
454 bh = head;
455 do {
456 get_bh(bh);
457 lock_buffer(bh);
458 bh = bh->b_this_page;
459
460 } while (bh != head);
461
462 ClearPagePrivate(page);
463 set_page_private(newpage, page_private(page));
464 set_page_private(page, 0);
465 put_page(page);
466 get_page(newpage);
467
468 bh = head;
469 do {
470 set_bh_page(bh, newpage, bh_offset(bh));
471 bh = bh->b_this_page;
472
473 } while (bh != head);
474
475 SetPagePrivate(newpage);
476
477 migrate_page_copy(newpage, page);
478
479 bh = head;
480 do {
481 unlock_buffer(bh);
482 put_bh(bh);
483 bh = bh->b_this_page;
484
485 } while (bh != head);
486
487 return 0;
488}
489EXPORT_SYMBOL(buffer_migrate_page);
David Howells93614012006-09-30 20:45:40 +0200490#endif
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700491
Christoph Lameter04e62a22006-06-23 02:03:38 -0700492/*
493 * Writeback a page to clean the dirty state
494 */
495static int writeout(struct address_space *mapping, struct page *page)
496{
497 struct writeback_control wbc = {
498 .sync_mode = WB_SYNC_NONE,
499 .nr_to_write = 1,
500 .range_start = 0,
501 .range_end = LLONG_MAX,
Christoph Lameter04e62a22006-06-23 02:03:38 -0700502 .for_reclaim = 1
503 };
504 int rc;
505
506 if (!mapping->a_ops->writepage)
507 /* No write method for the address space */
508 return -EINVAL;
509
510 if (!clear_page_dirty_for_io(page))
511 /* Someone else already triggered a write */
512 return -EAGAIN;
513
514 /*
515 * A dirty page may imply that the underlying filesystem has
516 * the page on some queue. So the page must be clean for
517 * migration. Writeout may mean we loose the lock and the
518 * page state is no longer what we checked for earlier.
519 * At this point we know that the migration attempt cannot
520 * be successful.
521 */
522 remove_migration_ptes(page, page);
523
524 rc = mapping->a_ops->writepage(page, &wbc);
Christoph Lameter04e62a22006-06-23 02:03:38 -0700525
526 if (rc != AOP_WRITEPAGE_ACTIVATE)
527 /* unlocked. Relock */
528 lock_page(page);
529
Hugh Dickinsbda85502008-11-19 15:36:36 -0800530 return (rc < 0) ? -EIO : -EAGAIN;
Christoph Lameter04e62a22006-06-23 02:03:38 -0700531}
532
533/*
534 * Default handling if a filesystem does not provide a migration function.
535 */
Christoph Lameter8351a6e2006-06-23 02:03:33 -0700536static int fallback_migrate_page(struct address_space *mapping,
537 struct page *newpage, struct page *page)
538{
Christoph Lameter04e62a22006-06-23 02:03:38 -0700539 if (PageDirty(page))
540 return writeout(mapping, page);
Christoph Lameter8351a6e2006-06-23 02:03:33 -0700541
542 /*
543 * Buffers may be managed in a filesystem specific way.
544 * We must have no buffers or drop them.
545 */
David Howells266cf652009-04-03 16:42:36 +0100546 if (page_has_private(page) &&
Christoph Lameter8351a6e2006-06-23 02:03:33 -0700547 !try_to_release_page(page, GFP_KERNEL))
548 return -EAGAIN;
549
550 return migrate_page(mapping, newpage, page);
551}
552
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700553/*
Christoph Lametere24f0b82006-06-23 02:03:51 -0700554 * Move a page to a newly allocated page
555 * The page is locked and all ptes have been successfully removed.
556 *
557 * The new page will have replaced the old page if this function
558 * is successful.
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700559 *
560 * Return value:
561 * < 0 - error code
562 * == 0 - success
Christoph Lametere24f0b82006-06-23 02:03:51 -0700563 */
Mel Gorman3fe20112010-05-24 14:32:20 -0700564static int move_to_new_page(struct page *newpage, struct page *page,
565 int remap_swapcache)
Christoph Lametere24f0b82006-06-23 02:03:51 -0700566{
567 struct address_space *mapping;
568 int rc;
569
570 /*
571 * Block others from accessing the page when we get around to
572 * establishing additional references. We are the only one
573 * holding a reference to the new page at this point.
574 */
Nick Piggin529ae9a2008-08-02 12:01:03 +0200575 if (!trylock_page(newpage))
Christoph Lametere24f0b82006-06-23 02:03:51 -0700576 BUG();
577
578 /* Prepare mapping for the new page.*/
579 newpage->index = page->index;
580 newpage->mapping = page->mapping;
Rik van Rielb2e18532008-10-18 20:26:30 -0700581 if (PageSwapBacked(page))
582 SetPageSwapBacked(newpage);
Christoph Lametere24f0b82006-06-23 02:03:51 -0700583
584 mapping = page_mapping(page);
585 if (!mapping)
586 rc = migrate_page(mapping, newpage, page);
587 else if (mapping->a_ops->migratepage)
588 /*
589 * Most pages have a mapping and most filesystems
590 * should provide a migration function. Anonymous
591 * pages are part of swap space which also has its
592 * own migration function. This is the most common
593 * path for page migration.
594 */
595 rc = mapping->a_ops->migratepage(mapping,
596 newpage, page);
597 else
598 rc = fallback_migrate_page(mapping, newpage, page);
599
Mel Gorman3fe20112010-05-24 14:32:20 -0700600 if (rc) {
Christoph Lametere24f0b82006-06-23 02:03:51 -0700601 newpage->mapping = NULL;
Mel Gorman3fe20112010-05-24 14:32:20 -0700602 } else {
603 if (remap_swapcache)
604 remove_migration_ptes(page, newpage);
605 }
Christoph Lametere24f0b82006-06-23 02:03:51 -0700606
607 unlock_page(newpage);
608
609 return rc;
610}
611
612/*
613 * Obtain the lock on page, remove all ptes and migrate the page
614 * to the newly allocated page in newpage.
615 */
Christoph Lameter95a402c2006-06-23 02:03:53 -0700616static int unmap_and_move(new_page_t get_new_page, unsigned long private,
Mel Gorman7f0f2492011-01-13 15:45:58 -0800617 struct page *page, int force, bool offlining, bool sync)
Christoph Lametere24f0b82006-06-23 02:03:51 -0700618{
619 int rc = 0;
Christoph Lameter742755a2006-06-23 02:03:55 -0700620 int *result = NULL;
621 struct page *newpage = get_new_page(page, private, &result);
Mel Gorman3fe20112010-05-24 14:32:20 -0700622 int remap_swapcache = 1;
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700623 int rcu_locked = 0;
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -0800624 int charge = 0;
KAMEZAWA Hiroyukie00e4312009-11-11 14:26:26 -0800625 struct mem_cgroup *mem = NULL;
Mel Gorman3f6c8272010-05-24 14:32:17 -0700626 struct anon_vma *anon_vma = NULL;
Christoph Lameter95a402c2006-06-23 02:03:53 -0700627
628 if (!newpage)
629 return -ENOMEM;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700630
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700631 if (page_count(page) == 1) {
Christoph Lametere24f0b82006-06-23 02:03:51 -0700632 /* page was freed from under us. So we are done. */
Christoph Lameter95a402c2006-06-23 02:03:53 -0700633 goto move_newpage;
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700634 }
Christoph Lametere24f0b82006-06-23 02:03:51 -0700635
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700636 /* prepare cgroup just returns 0 or -ENOMEM */
Christoph Lametere24f0b82006-06-23 02:03:51 -0700637 rc = -EAGAIN;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -0800638
Nick Piggin529ae9a2008-08-02 12:01:03 +0200639 if (!trylock_page(page)) {
Christoph Lametere24f0b82006-06-23 02:03:51 -0700640 if (!force)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700641 goto move_newpage;
Mel Gorman3e7d3442011-01-13 15:45:56 -0800642
643 /*
644 * It's not safe for direct compaction to call lock_page.
645 * For example, during page readahead pages are added locked
646 * to the LRU. Later, when the IO completes the pages are
647 * marked uptodate and unlocked. However, the queueing
648 * could be merging multiple pages for one bio (e.g.
649 * mpage_readpages). If an allocation happens for the
650 * second or third page, the process can end up locking
651 * the same page twice and deadlocking. Rather than
652 * trying to be clever about what pages can be locked,
653 * avoid the use of lock_page for direct compaction
654 * altogether.
655 */
656 if (current->flags & PF_MEMALLOC)
657 goto move_newpage;
658
Christoph Lametere24f0b82006-06-23 02:03:51 -0700659 lock_page(page);
660 }
661
Hugh Dickins62b61f62009-12-14 17:59:33 -0800662 /*
663 * Only memory hotplug's offline_pages() caller has locked out KSM,
664 * and can safely migrate a KSM page. The other cases have skipped
665 * PageKsm along with PageReserved - but it is only now when we have
666 * the page lock that we can be certain it will not go KSM beneath us
667 * (KSM will not upgrade a page from PageAnon to PageKsm when it sees
668 * its pagecount raised, but only here do we take the page lock which
669 * serializes that).
670 */
671 if (PageKsm(page) && !offlining) {
672 rc = -EBUSY;
673 goto unlock;
674 }
675
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -0800676 /* charge against new page */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -0700677 charge = mem_cgroup_prepare_migration(page, newpage, &mem);
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -0800678 if (charge == -ENOMEM) {
679 rc = -ENOMEM;
680 goto unlock;
681 }
682 BUG_ON(charge);
683
Christoph Lametere24f0b82006-06-23 02:03:51 -0700684 if (PageWriteback(page)) {
Mel Gorman77f1fe62011-01-13 15:45:57 -0800685 if (!force || !sync)
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -0800686 goto uncharge;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700687 wait_on_page_writeback(page);
688 }
Christoph Lametere24f0b82006-06-23 02:03:51 -0700689 /*
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700690 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
691 * we cannot notice that anon_vma is freed while we migrates a page.
692 * This rcu_read_lock() delays freeing anon_vma pointer until the end
693 * of migration. File cache pages are no problem because of page_lock()
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700694 * File Caches may use write_page() or lock_page() in migration, then,
695 * just care Anon page here.
Christoph Lametere24f0b82006-06-23 02:03:51 -0700696 */
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700697 if (PageAnon(page)) {
698 rcu_read_lock();
699 rcu_locked = 1;
Mel Gorman67b95092010-05-24 14:32:19 -0700700
Mel Gorman3fe20112010-05-24 14:32:20 -0700701 /* Determine how to safely use anon_vma */
702 if (!page_mapped(page)) {
703 if (!PageSwapCache(page))
704 goto rcu_unlock;
Mel Gorman67b95092010-05-24 14:32:19 -0700705
Mel Gorman3fe20112010-05-24 14:32:20 -0700706 /*
707 * We cannot be sure that the anon_vma of an unmapped
708 * swapcache page is safe to use because we don't
709 * know in advance if the VMA that this page belonged
710 * to still exists. If the VMA and others sharing the
711 * data have been freed, then the anon_vma could
712 * already be invalid.
713 *
714 * To avoid this possibility, swapcache pages get
715 * migrated but are not remapped when migration
716 * completes
717 */
718 remap_swapcache = 0;
719 } else {
720 /*
721 * Take a reference count on the anon_vma if the
722 * page is mapped so that it is guaranteed to
723 * exist when the page is remapped later
724 */
725 anon_vma = page_anon_vma(page);
Rik van Riel76545062010-08-09 17:18:41 -0700726 get_anon_vma(anon_vma);
Mel Gorman3fe20112010-05-24 14:32:20 -0700727 }
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700728 }
Shaohua Li62e1c552008-02-04 22:29:33 -0800729
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700730 /*
Shaohua Li62e1c552008-02-04 22:29:33 -0800731 * Corner case handling:
732 * 1. When a new swap-cache page is read into, it is added to the LRU
733 * and treated as swapcache but it has no rmap yet.
734 * Calling try_to_unmap() against a page->mapping==NULL page will
735 * trigger a BUG. So handle it here.
736 * 2. An orphaned page (see truncate_complete_page) might have
737 * fs-private metadata. The page can be picked up due to memory
738 * offlining. Everywhere else except page reclaim, the page is
739 * invisible to the vm, so the page can not be migrated. So try to
740 * free the metadata, so the page can be freed.
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700741 */
Shaohua Li62e1c552008-02-04 22:29:33 -0800742 if (!page->mapping) {
David Howells266cf652009-04-03 16:42:36 +0100743 if (!PageAnon(page) && page_has_private(page)) {
Shaohua Li62e1c552008-02-04 22:29:33 -0800744 /*
745 * Go direct to try_to_free_buffers() here because
746 * a) that's what try_to_release_page() would do anyway
747 * b) we may be under rcu_read_lock() here, so we can't
748 * use GFP_KERNEL which is what try_to_release_page()
749 * needs to be effective.
750 */
751 try_to_free_buffers(page);
Shaohua Liabfc3482009-09-21 17:01:19 -0700752 goto rcu_unlock;
Shaohua Li62e1c552008-02-04 22:29:33 -0800753 }
Shaohua Liabfc3482009-09-21 17:01:19 -0700754 goto skip_unmap;
Shaohua Li62e1c552008-02-04 22:29:33 -0800755 }
756
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700757 /* Establish migration ptes or remove ptes */
Andi Kleen14fa31b2009-09-16 11:50:10 +0200758 try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700759
Shaohua Liabfc3482009-09-21 17:01:19 -0700760skip_unmap:
Christoph Lametere6a15302006-06-25 05:46:49 -0700761 if (!page_mapped(page))
Mel Gorman3fe20112010-05-24 14:32:20 -0700762 rc = move_to_new_page(newpage, page, remap_swapcache);
Christoph Lametere24f0b82006-06-23 02:03:51 -0700763
Mel Gorman3fe20112010-05-24 14:32:20 -0700764 if (rc && remap_swapcache)
Christoph Lametere24f0b82006-06-23 02:03:51 -0700765 remove_migration_ptes(page, page);
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700766rcu_unlock:
Mel Gorman3f6c8272010-05-24 14:32:17 -0700767
768 /* Drop an anon_vma reference if we took one */
Rik van Riel76545062010-08-09 17:18:41 -0700769 if (anon_vma)
770 drop_anon_vma(anon_vma);
Mel Gorman3f6c8272010-05-24 14:32:17 -0700771
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700772 if (rcu_locked)
773 rcu_read_unlock();
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -0800774uncharge:
775 if (!charge)
776 mem_cgroup_end_migration(mem, page, newpage);
Christoph Lametere24f0b82006-06-23 02:03:51 -0700777unlock:
778 unlock_page(page);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700779
Christoph Lametere24f0b82006-06-23 02:03:51 -0700780 if (rc != -EAGAIN) {
Christoph Lameteraaa994b2006-06-23 02:03:52 -0700781 /*
782 * A page that has been migrated has all references
783 * removed and will be freed. A page that has not been
784 * migrated will have kepts its references and be
785 * restored.
786 */
787 list_del(&page->lru);
KOSAKI Motohiroa7312862009-09-21 17:01:37 -0700788 dec_zone_page_state(page, NR_ISOLATED_ANON +
Johannes Weiner6c0b1352009-09-21 17:02:59 -0700789 page_is_file_cache(page));
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700790 putback_lru_page(page);
Christoph Lametere24f0b82006-06-23 02:03:51 -0700791 }
Christoph Lameter95a402c2006-06-23 02:03:53 -0700792
793move_newpage:
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700794
Christoph Lameter95a402c2006-06-23 02:03:53 -0700795 /*
796 * Move the new page to the LRU. If migration was not successful
797 * then this will free the page.
798 */
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700799 putback_lru_page(newpage);
800
Christoph Lameter742755a2006-06-23 02:03:55 -0700801 if (result) {
802 if (rc)
803 *result = rc;
804 else
805 *result = page_to_nid(newpage);
806 }
Christoph Lametere24f0b82006-06-23 02:03:51 -0700807 return rc;
808}
809
810/*
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900811 * Counterpart of unmap_and_move_page() for hugepage migration.
812 *
813 * This function doesn't wait the completion of hugepage I/O
814 * because there is no race between I/O and migration for hugepage.
815 * Note that currently hugepage I/O occurs only in direct I/O
816 * where no lock is held and PG_writeback is irrelevant,
817 * and writeback status of all subpages are counted in the reference
818 * count of the head page (i.e. if all subpages of a 2MB hugepage are
819 * under direct I/O, the reference of the head page is 512 and a bit more.)
820 * This means that when we try to migrate hugepage whose subpages are
821 * doing direct I/O, some references remain after try_to_unmap() and
822 * hugepage migration fails without data corruption.
823 *
824 * There is also no race when direct I/O is issued on the page under migration,
825 * because then pte is replaced with migration swap entry and direct I/O code
826 * will wait in the page fault for migration to complete.
827 */
828static int unmap_and_move_huge_page(new_page_t get_new_page,
829 unsigned long private, struct page *hpage,
Mel Gorman7f0f2492011-01-13 15:45:58 -0800830 int force, bool offlining, bool sync)
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900831{
832 int rc = 0;
833 int *result = NULL;
834 struct page *new_hpage = get_new_page(hpage, private, &result);
835 int rcu_locked = 0;
836 struct anon_vma *anon_vma = NULL;
837
838 if (!new_hpage)
839 return -ENOMEM;
840
841 rc = -EAGAIN;
842
843 if (!trylock_page(hpage)) {
Mel Gorman77f1fe62011-01-13 15:45:57 -0800844 if (!force || !sync)
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900845 goto out;
846 lock_page(hpage);
847 }
848
849 if (PageAnon(hpage)) {
850 rcu_read_lock();
851 rcu_locked = 1;
852
853 if (page_mapped(hpage)) {
854 anon_vma = page_anon_vma(hpage);
855 atomic_inc(&anon_vma->external_refcount);
856 }
857 }
858
859 try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
860
861 if (!page_mapped(hpage))
862 rc = move_to_new_page(new_hpage, hpage, 1);
863
864 if (rc)
865 remove_migration_ptes(hpage, hpage);
866
867 if (anon_vma && atomic_dec_and_lock(&anon_vma->external_refcount,
868 &anon_vma->lock)) {
869 int empty = list_empty(&anon_vma->head);
870 spin_unlock(&anon_vma->lock);
871 if (empty)
872 anon_vma_free(anon_vma);
873 }
874
875 if (rcu_locked)
876 rcu_read_unlock();
877out:
878 unlock_page(hpage);
879
880 if (rc != -EAGAIN) {
881 list_del(&hpage->lru);
882 put_page(hpage);
883 }
884
885 put_page(new_hpage);
886
887 if (result) {
888 if (rc)
889 *result = rc;
890 else
891 *result = page_to_nid(new_hpage);
892 }
893 return rc;
894}
895
896/*
Christoph Lameterb20a3502006-03-22 00:09:12 -0800897 * migrate_pages
898 *
Christoph Lameter95a402c2006-06-23 02:03:53 -0700899 * The function takes one list of pages to migrate and a function
900 * that determines from the page to be migrated and the private data
901 * the target of the move and allocates the page.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800902 *
903 * The function returns after 10 attempts or if no pages
904 * are movable anymore because to has become empty
Minchan Kimcf608ac2010-10-26 14:21:29 -0700905 * or no retryable pages exist anymore.
906 * Caller should call putback_lru_pages to return pages to the LRU
907 * or free list.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800908 *
Christoph Lameter95a402c2006-06-23 02:03:53 -0700909 * Return: Number of pages not migrated or error code.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800910 */
Christoph Lameter95a402c2006-06-23 02:03:53 -0700911int migrate_pages(struct list_head *from,
Mel Gorman7f0f2492011-01-13 15:45:58 -0800912 new_page_t get_new_page, unsigned long private, bool offlining,
Mel Gorman77f1fe62011-01-13 15:45:57 -0800913 bool sync)
Christoph Lameterb20a3502006-03-22 00:09:12 -0800914{
Christoph Lametere24f0b82006-06-23 02:03:51 -0700915 int retry = 1;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800916 int nr_failed = 0;
917 int pass = 0;
918 struct page *page;
919 struct page *page2;
920 int swapwrite = current->flags & PF_SWAPWRITE;
921 int rc;
922
923 if (!swapwrite)
924 current->flags |= PF_SWAPWRITE;
925
Christoph Lametere24f0b82006-06-23 02:03:51 -0700926 for(pass = 0; pass < 10 && retry; pass++) {
927 retry = 0;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800928
Christoph Lametere24f0b82006-06-23 02:03:51 -0700929 list_for_each_entry_safe(page, page2, from, lru) {
Christoph Lametere24f0b82006-06-23 02:03:51 -0700930 cond_resched();
Christoph Lameterb20a3502006-03-22 00:09:12 -0800931
Christoph Lameter95a402c2006-06-23 02:03:53 -0700932 rc = unmap_and_move(get_new_page, private,
Mel Gorman77f1fe62011-01-13 15:45:57 -0800933 page, pass > 2, offlining,
934 sync);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800935
Christoph Lametere24f0b82006-06-23 02:03:51 -0700936 switch(rc) {
Christoph Lameter95a402c2006-06-23 02:03:53 -0700937 case -ENOMEM:
938 goto out;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700939 case -EAGAIN:
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700940 retry++;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700941 break;
942 case 0:
Christoph Lametere24f0b82006-06-23 02:03:51 -0700943 break;
944 default:
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700945 /* Permanent failure */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700946 nr_failed++;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700947 break;
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700948 }
Christoph Lameterb20a3502006-03-22 00:09:12 -0800949 }
950 }
Christoph Lameter95a402c2006-06-23 02:03:53 -0700951 rc = 0;
952out:
Christoph Lameterb20a3502006-03-22 00:09:12 -0800953 if (!swapwrite)
954 current->flags &= ~PF_SWAPWRITE;
955
Christoph Lameter95a402c2006-06-23 02:03:53 -0700956 if (rc)
957 return rc;
958
Christoph Lameterb20a3502006-03-22 00:09:12 -0800959 return nr_failed + retry;
960}
961
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900962int migrate_huge_pages(struct list_head *from,
Mel Gorman7f0f2492011-01-13 15:45:58 -0800963 new_page_t get_new_page, unsigned long private, bool offlining,
Mel Gorman77f1fe62011-01-13 15:45:57 -0800964 bool sync)
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900965{
966 int retry = 1;
967 int nr_failed = 0;
968 int pass = 0;
969 struct page *page;
970 struct page *page2;
971 int rc;
972
973 for (pass = 0; pass < 10 && retry; pass++) {
974 retry = 0;
975
976 list_for_each_entry_safe(page, page2, from, lru) {
977 cond_resched();
978
979 rc = unmap_and_move_huge_page(get_new_page,
Mel Gorman77f1fe62011-01-13 15:45:57 -0800980 private, page, pass > 2, offlining,
981 sync);
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900982
983 switch(rc) {
984 case -ENOMEM:
985 goto out;
986 case -EAGAIN:
987 retry++;
988 break;
989 case 0:
990 break;
991 default:
992 /* Permanent failure */
993 nr_failed++;
994 break;
995 }
996 }
997 }
998 rc = 0;
999out:
1000
1001 list_for_each_entry_safe(page, page2, from, lru)
1002 put_page(page);
1003
1004 if (rc)
1005 return rc;
1006
1007 return nr_failed + retry;
1008}
1009
Christoph Lameter742755a2006-06-23 02:03:55 -07001010#ifdef CONFIG_NUMA
1011/*
1012 * Move a list of individual pages
1013 */
1014struct page_to_node {
1015 unsigned long addr;
1016 struct page *page;
1017 int node;
1018 int status;
1019};
1020
1021static struct page *new_page_node(struct page *p, unsigned long private,
1022 int **result)
1023{
1024 struct page_to_node *pm = (struct page_to_node *)private;
1025
1026 while (pm->node != MAX_NUMNODES && pm->page != p)
1027 pm++;
1028
1029 if (pm->node == MAX_NUMNODES)
1030 return NULL;
1031
1032 *result = &pm->status;
1033
Mel Gorman6484eb32009-06-16 15:31:54 -07001034 return alloc_pages_exact_node(pm->node,
Mel Gorman769848c2007-07-17 04:03:05 -07001035 GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0);
Christoph Lameter742755a2006-06-23 02:03:55 -07001036}
1037
1038/*
1039 * Move a set of pages as indicated in the pm array. The addr
1040 * field must be set to the virtual address of the page to be moved
1041 * and the node number must contain a valid target node.
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001042 * The pm array ends with node = MAX_NUMNODES.
Christoph Lameter742755a2006-06-23 02:03:55 -07001043 */
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001044static int do_move_page_to_node_array(struct mm_struct *mm,
1045 struct page_to_node *pm,
1046 int migrate_all)
Christoph Lameter742755a2006-06-23 02:03:55 -07001047{
1048 int err;
1049 struct page_to_node *pp;
1050 LIST_HEAD(pagelist);
1051
1052 down_read(&mm->mmap_sem);
1053
1054 /*
1055 * Build a list of pages to migrate
1056 */
Christoph Lameter742755a2006-06-23 02:03:55 -07001057 for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
1058 struct vm_area_struct *vma;
1059 struct page *page;
1060
Christoph Lameter742755a2006-06-23 02:03:55 -07001061 err = -EFAULT;
1062 vma = find_vma(mm, pp->addr);
Gleb Natapov70384dc2010-10-26 14:22:07 -07001063 if (!vma || pp->addr < vma->vm_start || !vma_migratable(vma))
Christoph Lameter742755a2006-06-23 02:03:55 -07001064 goto set_status;
1065
1066 page = follow_page(vma, pp->addr, FOLL_GET);
Linus Torvalds89f5b7d2008-06-20 11:18:25 -07001067
1068 err = PTR_ERR(page);
1069 if (IS_ERR(page))
1070 goto set_status;
1071
Christoph Lameter742755a2006-06-23 02:03:55 -07001072 err = -ENOENT;
1073 if (!page)
1074 goto set_status;
1075
Hugh Dickins62b61f62009-12-14 17:59:33 -08001076 /* Use PageReserved to check for zero page */
1077 if (PageReserved(page) || PageKsm(page))
Christoph Lameter742755a2006-06-23 02:03:55 -07001078 goto put_and_set;
1079
1080 pp->page = page;
1081 err = page_to_nid(page);
1082
1083 if (err == pp->node)
1084 /*
1085 * Node already in the right place
1086 */
1087 goto put_and_set;
1088
1089 err = -EACCES;
1090 if (page_mapcount(page) > 1 &&
1091 !migrate_all)
1092 goto put_and_set;
1093
Nick Piggin62695a82008-10-18 20:26:09 -07001094 err = isolate_lru_page(page);
KOSAKI Motohiro6d9c2852009-12-14 17:58:11 -08001095 if (!err) {
Nick Piggin62695a82008-10-18 20:26:09 -07001096 list_add_tail(&page->lru, &pagelist);
KOSAKI Motohiro6d9c2852009-12-14 17:58:11 -08001097 inc_zone_page_state(page, NR_ISOLATED_ANON +
1098 page_is_file_cache(page));
1099 }
Christoph Lameter742755a2006-06-23 02:03:55 -07001100put_and_set:
1101 /*
1102 * Either remove the duplicate refcount from
1103 * isolate_lru_page() or drop the page ref if it was
1104 * not isolated.
1105 */
1106 put_page(page);
1107set_status:
1108 pp->status = err;
1109 }
1110
Brice Gogline78bbfa2008-10-18 20:27:15 -07001111 err = 0;
Minchan Kimcf608ac2010-10-26 14:21:29 -07001112 if (!list_empty(&pagelist)) {
Christoph Lameter742755a2006-06-23 02:03:55 -07001113 err = migrate_pages(&pagelist, new_page_node,
Mel Gorman77f1fe62011-01-13 15:45:57 -08001114 (unsigned long)pm, 0, true);
Minchan Kimcf608ac2010-10-26 14:21:29 -07001115 if (err)
1116 putback_lru_pages(&pagelist);
1117 }
Christoph Lameter742755a2006-06-23 02:03:55 -07001118
1119 up_read(&mm->mmap_sem);
1120 return err;
1121}
1122
1123/*
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001124 * Migrate an array of page address onto an array of nodes and fill
1125 * the corresponding array of status.
1126 */
1127static int do_pages_move(struct mm_struct *mm, struct task_struct *task,
1128 unsigned long nr_pages,
1129 const void __user * __user *pages,
1130 const int __user *nodes,
1131 int __user *status, int flags)
1132{
Brice Goglin3140a222009-01-06 14:38:57 -08001133 struct page_to_node *pm;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001134 nodemask_t task_nodes;
Brice Goglin3140a222009-01-06 14:38:57 -08001135 unsigned long chunk_nr_pages;
1136 unsigned long chunk_start;
1137 int err;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001138
1139 task_nodes = cpuset_mems_allowed(task);
1140
Brice Goglin3140a222009-01-06 14:38:57 -08001141 err = -ENOMEM;
1142 pm = (struct page_to_node *)__get_free_page(GFP_KERNEL);
1143 if (!pm)
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001144 goto out;
Brice Goglin35282a22009-06-16 15:32:43 -07001145
1146 migrate_prep();
1147
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001148 /*
Brice Goglin3140a222009-01-06 14:38:57 -08001149 * Store a chunk of page_to_node array in a page,
1150 * but keep the last one as a marker
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001151 */
Brice Goglin3140a222009-01-06 14:38:57 -08001152 chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001153
Brice Goglin3140a222009-01-06 14:38:57 -08001154 for (chunk_start = 0;
1155 chunk_start < nr_pages;
1156 chunk_start += chunk_nr_pages) {
1157 int j;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001158
Brice Goglin3140a222009-01-06 14:38:57 -08001159 if (chunk_start + chunk_nr_pages > nr_pages)
1160 chunk_nr_pages = nr_pages - chunk_start;
1161
1162 /* fill the chunk pm with addrs and nodes from user-space */
1163 for (j = 0; j < chunk_nr_pages; j++) {
1164 const void __user *p;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001165 int node;
1166
Brice Goglin3140a222009-01-06 14:38:57 -08001167 err = -EFAULT;
1168 if (get_user(p, pages + j + chunk_start))
1169 goto out_pm;
1170 pm[j].addr = (unsigned long) p;
1171
1172 if (get_user(node, nodes + j + chunk_start))
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001173 goto out_pm;
1174
1175 err = -ENODEV;
Linus Torvalds6f5a55f2010-02-05 16:16:50 -08001176 if (node < 0 || node >= MAX_NUMNODES)
1177 goto out_pm;
1178
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001179 if (!node_state(node, N_HIGH_MEMORY))
1180 goto out_pm;
1181
1182 err = -EACCES;
1183 if (!node_isset(node, task_nodes))
1184 goto out_pm;
1185
Brice Goglin3140a222009-01-06 14:38:57 -08001186 pm[j].node = node;
1187 }
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001188
Brice Goglin3140a222009-01-06 14:38:57 -08001189 /* End marker for this chunk */
1190 pm[chunk_nr_pages].node = MAX_NUMNODES;
1191
1192 /* Migrate this chunk */
1193 err = do_move_page_to_node_array(mm, pm,
1194 flags & MPOL_MF_MOVE_ALL);
1195 if (err < 0)
1196 goto out_pm;
1197
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001198 /* Return status information */
Brice Goglin3140a222009-01-06 14:38:57 -08001199 for (j = 0; j < chunk_nr_pages; j++)
1200 if (put_user(pm[j].status, status + j + chunk_start)) {
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001201 err = -EFAULT;
Brice Goglin3140a222009-01-06 14:38:57 -08001202 goto out_pm;
1203 }
1204 }
1205 err = 0;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001206
1207out_pm:
Brice Goglin3140a222009-01-06 14:38:57 -08001208 free_page((unsigned long)pm);
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001209out:
1210 return err;
1211}
1212
1213/*
Brice Goglin2f007e72008-10-18 20:27:16 -07001214 * Determine the nodes of an array of pages and store it in an array of status.
Christoph Lameter742755a2006-06-23 02:03:55 -07001215 */
Brice Goglin80bba122008-12-09 13:14:23 -08001216static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
1217 const void __user **pages, int *status)
Christoph Lameter742755a2006-06-23 02:03:55 -07001218{
Brice Goglin2f007e72008-10-18 20:27:16 -07001219 unsigned long i;
Brice Goglin2f007e72008-10-18 20:27:16 -07001220
Christoph Lameter742755a2006-06-23 02:03:55 -07001221 down_read(&mm->mmap_sem);
1222
Brice Goglin2f007e72008-10-18 20:27:16 -07001223 for (i = 0; i < nr_pages; i++) {
Brice Goglin80bba122008-12-09 13:14:23 -08001224 unsigned long addr = (unsigned long)(*pages);
Christoph Lameter742755a2006-06-23 02:03:55 -07001225 struct vm_area_struct *vma;
1226 struct page *page;
KOSAKI Motohiroc095adb2008-12-16 16:06:43 +09001227 int err = -EFAULT;
Brice Goglin2f007e72008-10-18 20:27:16 -07001228
1229 vma = find_vma(mm, addr);
Gleb Natapov70384dc2010-10-26 14:22:07 -07001230 if (!vma || addr < vma->vm_start)
Christoph Lameter742755a2006-06-23 02:03:55 -07001231 goto set_status;
1232
Brice Goglin2f007e72008-10-18 20:27:16 -07001233 page = follow_page(vma, addr, 0);
Linus Torvalds89f5b7d2008-06-20 11:18:25 -07001234
1235 err = PTR_ERR(page);
1236 if (IS_ERR(page))
1237 goto set_status;
1238
Christoph Lameter742755a2006-06-23 02:03:55 -07001239 err = -ENOENT;
1240 /* Use PageReserved to check for zero page */
Hugh Dickins62b61f62009-12-14 17:59:33 -08001241 if (!page || PageReserved(page) || PageKsm(page))
Christoph Lameter742755a2006-06-23 02:03:55 -07001242 goto set_status;
1243
1244 err = page_to_nid(page);
1245set_status:
Brice Goglin80bba122008-12-09 13:14:23 -08001246 *status = err;
1247
1248 pages++;
1249 status++;
1250 }
1251
1252 up_read(&mm->mmap_sem);
1253}
1254
1255/*
1256 * Determine the nodes of a user array of pages and store it in
1257 * a user array of status.
1258 */
1259static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
1260 const void __user * __user *pages,
1261 int __user *status)
1262{
1263#define DO_PAGES_STAT_CHUNK_NR 16
1264 const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
1265 int chunk_status[DO_PAGES_STAT_CHUNK_NR];
Brice Goglin80bba122008-12-09 13:14:23 -08001266
H. Peter Anvin87b8d1a2010-02-18 16:13:40 -08001267 while (nr_pages) {
1268 unsigned long chunk_nr;
Brice Goglin80bba122008-12-09 13:14:23 -08001269
H. Peter Anvin87b8d1a2010-02-18 16:13:40 -08001270 chunk_nr = nr_pages;
1271 if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
1272 chunk_nr = DO_PAGES_STAT_CHUNK_NR;
1273
1274 if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
1275 break;
Brice Goglin80bba122008-12-09 13:14:23 -08001276
1277 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
1278
H. Peter Anvin87b8d1a2010-02-18 16:13:40 -08001279 if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
1280 break;
Christoph Lameter742755a2006-06-23 02:03:55 -07001281
H. Peter Anvin87b8d1a2010-02-18 16:13:40 -08001282 pages += chunk_nr;
1283 status += chunk_nr;
1284 nr_pages -= chunk_nr;
1285 }
1286 return nr_pages ? -EFAULT : 0;
Christoph Lameter742755a2006-06-23 02:03:55 -07001287}
1288
1289/*
1290 * Move a list of pages in the address space of the currently executing
1291 * process.
1292 */
Heiko Carstens938bb9f2009-01-14 14:14:30 +01001293SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
1294 const void __user * __user *, pages,
1295 const int __user *, nodes,
1296 int __user *, status, int, flags)
Christoph Lameter742755a2006-06-23 02:03:55 -07001297{
David Howellsc69e8d92008-11-14 10:39:19 +11001298 const struct cred *cred = current_cred(), *tcred;
Christoph Lameter742755a2006-06-23 02:03:55 -07001299 struct task_struct *task;
Christoph Lameter742755a2006-06-23 02:03:55 -07001300 struct mm_struct *mm;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001301 int err;
Christoph Lameter742755a2006-06-23 02:03:55 -07001302
1303 /* Check flags */
1304 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
1305 return -EINVAL;
1306
1307 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1308 return -EPERM;
1309
1310 /* Find the mm_struct */
1311 read_lock(&tasklist_lock);
Pavel Emelyanov228ebcb2007-10-18 23:40:16 -07001312 task = pid ? find_task_by_vpid(pid) : current;
Christoph Lameter742755a2006-06-23 02:03:55 -07001313 if (!task) {
1314 read_unlock(&tasklist_lock);
1315 return -ESRCH;
1316 }
1317 mm = get_task_mm(task);
1318 read_unlock(&tasklist_lock);
1319
1320 if (!mm)
1321 return -EINVAL;
1322
1323 /*
1324 * Check if this process has the right to modify the specified
1325 * process. The right exists if the process has administrative
1326 * capabilities, superuser privileges or the same
1327 * userid as the target process.
1328 */
David Howellsc69e8d92008-11-14 10:39:19 +11001329 rcu_read_lock();
1330 tcred = __task_cred(task);
David Howellsb6dff3e2008-11-14 10:39:16 +11001331 if (cred->euid != tcred->suid && cred->euid != tcred->uid &&
1332 cred->uid != tcred->suid && cred->uid != tcred->uid &&
Christoph Lameter742755a2006-06-23 02:03:55 -07001333 !capable(CAP_SYS_NICE)) {
David Howellsc69e8d92008-11-14 10:39:19 +11001334 rcu_read_unlock();
Christoph Lameter742755a2006-06-23 02:03:55 -07001335 err = -EPERM;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001336 goto out;
Christoph Lameter742755a2006-06-23 02:03:55 -07001337 }
David Howellsc69e8d92008-11-14 10:39:19 +11001338 rcu_read_unlock();
Christoph Lameter742755a2006-06-23 02:03:55 -07001339
David Quigley86c3a762006-06-23 02:04:02 -07001340 err = security_task_movememory(task);
1341 if (err)
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001342 goto out;
David Quigley86c3a762006-06-23 02:04:02 -07001343
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001344 if (nodes) {
1345 err = do_pages_move(mm, task, nr_pages, pages, nodes, status,
1346 flags);
1347 } else {
Brice Goglin2f007e72008-10-18 20:27:16 -07001348 err = do_pages_stat(mm, nr_pages, pages, status);
Brice Goglin2f007e72008-10-18 20:27:16 -07001349 }
David Quigley86c3a762006-06-23 02:04:02 -07001350
Christoph Lameter742755a2006-06-23 02:03:55 -07001351out:
Christoph Lameter742755a2006-06-23 02:03:55 -07001352 mmput(mm);
1353 return err;
1354}
Christoph Lameter742755a2006-06-23 02:03:55 -07001355
Christoph Lameter7b2259b2006-06-25 05:46:48 -07001356/*
1357 * Call migration functions in the vma_ops that may prepare
1358 * memory in a vm for migration. migration functions may perform
1359 * the migration for vmas that do not have an underlying page struct.
1360 */
1361int migrate_vmas(struct mm_struct *mm, const nodemask_t *to,
1362 const nodemask_t *from, unsigned long flags)
1363{
1364 struct vm_area_struct *vma;
1365 int err = 0;
1366
Daisuke Nishimura1001c9f2009-02-11 13:04:18 -08001367 for (vma = mm->mmap; vma && !err; vma = vma->vm_next) {
Christoph Lameter7b2259b2006-06-25 05:46:48 -07001368 if (vma->vm_ops && vma->vm_ops->migrate) {
1369 err = vma->vm_ops->migrate(vma, to, from, flags);
1370 if (err)
1371 break;
1372 }
1373 }
1374 return err;
1375}
Gerald Schaefer83d16742008-07-23 21:28:22 -07001376#endif