blob: 35e454189966cbe42eab5e2172c7d64746d864c8 [file] [log] [blame]
Christoph Lameterb20a3502006-03-22 00:09:12 -08001/*
2 * Memory Migration functionality - linux/mm/migration.c
3 *
4 * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
5 *
6 * Page migration was first developed in the context of the memory hotplug
7 * project. The main authors of the migration code are:
8 *
9 * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
10 * Hirokazu Takahashi <taka@valinux.co.jp>
11 * Dave Hansen <haveblue@us.ibm.com>
Christoph Lametercde53532008-07-04 09:59:22 -070012 * Christoph Lameter
Christoph Lameterb20a3502006-03-22 00:09:12 -080013 */
14
15#include <linux/migrate.h>
16#include <linux/module.h>
17#include <linux/swap.h>
Christoph Lameter06972122006-06-23 02:03:35 -070018#include <linux/swapops.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080019#include <linux/pagemap.h>
Christoph Lametere23ca002006-04-10 22:52:57 -070020#include <linux/buffer_head.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080021#include <linux/mm_inline.h>
Pavel Emelyanovb4888932007-10-18 23:40:14 -070022#include <linux/nsproxy.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080023#include <linux/pagevec.h>
Hugh Dickinse9995ef2009-12-14 17:59:31 -080024#include <linux/ksm.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080025#include <linux/rmap.h>
26#include <linux/topology.h>
27#include <linux/cpu.h>
28#include <linux/cpuset.h>
Christoph Lameter04e62a22006-06-23 02:03:38 -070029#include <linux/writeback.h>
Christoph Lameter742755a2006-06-23 02:03:55 -070030#include <linux/mempolicy.h>
31#include <linux/vmalloc.h>
David Quigley86c3a762006-06-23 02:04:02 -070032#include <linux/security.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080033#include <linux/memcontrol.h>
Adrian Bunk4f5ca262008-07-23 21:27:02 -070034#include <linux/syscalls.h>
Naoya Horiguchi290408d2010-09-08 10:19:35 +090035#include <linux/hugetlb.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090036#include <linux/gfp.h>
Christoph Lameterb20a3502006-03-22 00:09:12 -080037
38#include "internal.h"
39
Christoph Lameterb20a3502006-03-22 00:09:12 -080040#define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
41
42/*
Christoph Lameter742755a2006-06-23 02:03:55 -070043 * migrate_prep() needs to be called before we start compiling a list of pages
Mel Gorman748446b2010-05-24 14:32:27 -070044 * to be migrated using isolate_lru_page(). If scheduling work on other CPUs is
45 * undesirable, use migrate_prep_local()
Christoph Lameterb20a3502006-03-22 00:09:12 -080046 */
47int migrate_prep(void)
48{
Christoph Lameterb20a3502006-03-22 00:09:12 -080049 /*
50 * Clear the LRU lists so pages can be isolated.
51 * Note that pages may be moved off the LRU after we have
52 * drained them. Those pages will fail to migrate like other
53 * pages that may be busy.
54 */
55 lru_add_drain_all();
56
57 return 0;
58}
59
Mel Gorman748446b2010-05-24 14:32:27 -070060/* Do the necessary work of migrate_prep but not if it involves other CPUs */
61int migrate_prep_local(void)
62{
63 lru_add_drain();
64
65 return 0;
66}
67
Christoph Lameterb20a3502006-03-22 00:09:12 -080068/*
Lee Schermerhorn894bc312008-10-18 20:26:39 -070069 * Add isolated pages on the list back to the LRU under page lock
70 * to avoid leaking evictable pages back onto unevictable list.
Christoph Lameterb20a3502006-03-22 00:09:12 -080071 */
Minchan Kime13861d2010-05-24 14:31:59 -070072void putback_lru_pages(struct list_head *l)
Christoph Lameterb20a3502006-03-22 00:09:12 -080073{
74 struct page *page;
75 struct page *page2;
Christoph Lameterb20a3502006-03-22 00:09:12 -080076
77 list_for_each_entry_safe(page, page2, l, lru) {
Christoph Lametere24f0b82006-06-23 02:03:51 -070078 list_del(&page->lru);
KOSAKI Motohiroa7312862009-09-21 17:01:37 -070079 dec_zone_page_state(page, NR_ISOLATED_ANON +
Johannes Weiner6c0b1352009-09-21 17:02:59 -070080 page_is_file_cache(page));
Lee Schermerhorn894bc312008-10-18 20:26:39 -070081 putback_lru_page(page);
Christoph Lameterb20a3502006-03-22 00:09:12 -080082 }
Christoph Lameterb20a3502006-03-22 00:09:12 -080083}
84
Christoph Lameter06972122006-06-23 02:03:35 -070085/*
86 * Restore a potential migration pte to a working pte entry
87 */
Hugh Dickinse9995ef2009-12-14 17:59:31 -080088static int remove_migration_pte(struct page *new, struct vm_area_struct *vma,
89 unsigned long addr, void *old)
Christoph Lameter06972122006-06-23 02:03:35 -070090{
91 struct mm_struct *mm = vma->vm_mm;
92 swp_entry_t entry;
93 pgd_t *pgd;
94 pud_t *pud;
95 pmd_t *pmd;
96 pte_t *ptep, pte;
97 spinlock_t *ptl;
98
Naoya Horiguchi290408d2010-09-08 10:19:35 +090099 if (unlikely(PageHuge(new))) {
100 ptep = huge_pte_offset(mm, addr);
101 if (!ptep)
102 goto out;
103 ptl = &mm->page_table_lock;
104 } else {
105 pgd = pgd_offset(mm, addr);
106 if (!pgd_present(*pgd))
107 goto out;
Christoph Lameter06972122006-06-23 02:03:35 -0700108
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900109 pud = pud_offset(pgd, addr);
110 if (!pud_present(*pud))
111 goto out;
Christoph Lameter06972122006-06-23 02:03:35 -0700112
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900113 pmd = pmd_offset(pud, addr);
114 if (!pmd_present(*pmd))
115 goto out;
Christoph Lameter06972122006-06-23 02:03:35 -0700116
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900117 ptep = pte_offset_map(pmd, addr);
Christoph Lameter06972122006-06-23 02:03:35 -0700118
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900119 if (!is_swap_pte(*ptep)) {
120 pte_unmap(ptep);
121 goto out;
122 }
Christoph Lameter06972122006-06-23 02:03:35 -0700123
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900124 ptl = pte_lockptr(mm, pmd);
125 }
126
Christoph Lameter06972122006-06-23 02:03:35 -0700127 spin_lock(ptl);
128 pte = *ptep;
129 if (!is_swap_pte(pte))
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800130 goto unlock;
Christoph Lameter06972122006-06-23 02:03:35 -0700131
132 entry = pte_to_swp_entry(pte);
133
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800134 if (!is_migration_entry(entry) ||
135 migration_entry_to_page(entry) != old)
136 goto unlock;
Christoph Lameter06972122006-06-23 02:03:35 -0700137
Christoph Lameter06972122006-06-23 02:03:35 -0700138 get_page(new);
139 pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
140 if (is_write_migration_entry(entry))
141 pte = pte_mkwrite(pte);
Andi Kleen3ef8fd72010-10-11 16:03:21 +0200142#ifdef CONFIG_HUGETLB_PAGE
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900143 if (PageHuge(new))
144 pte = pte_mkhuge(pte);
Andi Kleen3ef8fd72010-10-11 16:03:21 +0200145#endif
KAMEZAWA Hiroyuki97ee0522007-10-16 01:25:43 -0700146 flush_cache_page(vma, addr, pte_pfn(pte));
Christoph Lameter06972122006-06-23 02:03:35 -0700147 set_pte_at(mm, addr, ptep, pte);
Christoph Lameter04e62a22006-06-23 02:03:38 -0700148
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900149 if (PageHuge(new)) {
150 if (PageAnon(new))
151 hugepage_add_anon_rmap(new, vma, addr);
152 else
153 page_dup_rmap(new);
154 } else if (PageAnon(new))
Christoph Lameter04e62a22006-06-23 02:03:38 -0700155 page_add_anon_rmap(new, vma, addr);
156 else
157 page_add_file_rmap(new);
158
159 /* No need to invalidate - it was non-present before */
Russell King4b3073e2009-12-18 16:40:18 +0000160 update_mmu_cache(vma, addr, ptep);
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800161unlock:
Christoph Lameter06972122006-06-23 02:03:35 -0700162 pte_unmap_unlock(ptep, ptl);
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800163out:
164 return SWAP_AGAIN;
Christoph Lameter06972122006-06-23 02:03:35 -0700165}
166
167/*
Christoph Lameter04e62a22006-06-23 02:03:38 -0700168 * Get rid of all migration entries and replace them by
169 * references to the indicated page.
170 */
171static void remove_migration_ptes(struct page *old, struct page *new)
172{
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800173 rmap_walk(new, remove_migration_pte, old);
Christoph Lameter04e62a22006-06-23 02:03:38 -0700174}
175
176/*
Christoph Lameter06972122006-06-23 02:03:35 -0700177 * Something used the pte of a page under migration. We need to
178 * get to the page and wait until migration is finished.
179 * When we return from this function the fault will be retried.
180 *
181 * This function is called from do_swap_page().
182 */
183void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
184 unsigned long address)
185{
186 pte_t *ptep, pte;
187 spinlock_t *ptl;
188 swp_entry_t entry;
189 struct page *page;
190
191 ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
192 pte = *ptep;
193 if (!is_swap_pte(pte))
194 goto out;
195
196 entry = pte_to_swp_entry(pte);
197 if (!is_migration_entry(entry))
198 goto out;
199
200 page = migration_entry_to_page(entry);
201
Nick Piggine2867812008-07-25 19:45:30 -0700202 /*
203 * Once radix-tree replacement of page migration started, page_count
204 * *must* be zero. And, we don't want to call wait_on_page_locked()
205 * against a page without get_page().
206 * So, we use get_page_unless_zero(), here. Even failed, page fault
207 * will occur again.
208 */
209 if (!get_page_unless_zero(page))
210 goto out;
Christoph Lameter06972122006-06-23 02:03:35 -0700211 pte_unmap_unlock(ptep, ptl);
212 wait_on_page_locked(page);
213 put_page(page);
214 return;
215out:
216 pte_unmap_unlock(ptep, ptl);
217}
218
Christoph Lameterb20a3502006-03-22 00:09:12 -0800219/*
Christoph Lameterc3fcf8a2006-06-23 02:03:32 -0700220 * Replace the page in the mapping.
Christoph Lameter5b5c7122006-06-23 02:03:29 -0700221 *
222 * The number of remaining references must be:
223 * 1 for anonymous pages without a mapping
224 * 2 for pages with a mapping
David Howells266cf652009-04-03 16:42:36 +0100225 * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800226 */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700227static int migrate_page_move_mapping(struct address_space *mapping,
228 struct page *newpage, struct page *page)
Christoph Lameterb20a3502006-03-22 00:09:12 -0800229{
Nick Piggine2867812008-07-25 19:45:30 -0700230 int expected_count;
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800231 void **pslot;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800232
Christoph Lameter6c5240a2006-06-23 02:03:37 -0700233 if (!mapping) {
Christoph Lameter0e8c7d02007-04-23 14:41:09 -0700234 /* Anonymous page without mapping */
Christoph Lameter6c5240a2006-06-23 02:03:37 -0700235 if (page_count(page) != 1)
236 return -EAGAIN;
237 return 0;
238 }
239
Nick Piggin19fd6232008-07-25 19:45:32 -0700240 spin_lock_irq(&mapping->tree_lock);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800241
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800242 pslot = radix_tree_lookup_slot(&mapping->page_tree,
243 page_index(page));
Christoph Lameterb20a3502006-03-22 00:09:12 -0800244
Johannes Weineredcf4742009-09-21 17:02:59 -0700245 expected_count = 2 + page_has_private(page);
Nick Piggine2867812008-07-25 19:45:30 -0700246 if (page_count(page) != expected_count ||
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800247 (struct page *)radix_tree_deref_slot(pslot) != page) {
Nick Piggin19fd6232008-07-25 19:45:32 -0700248 spin_unlock_irq(&mapping->tree_lock);
Christoph Lametere23ca002006-04-10 22:52:57 -0700249 return -EAGAIN;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800250 }
251
Nick Piggine2867812008-07-25 19:45:30 -0700252 if (!page_freeze_refs(page, expected_count)) {
Nick Piggin19fd6232008-07-25 19:45:32 -0700253 spin_unlock_irq(&mapping->tree_lock);
Nick Piggine2867812008-07-25 19:45:30 -0700254 return -EAGAIN;
255 }
256
Christoph Lameterb20a3502006-03-22 00:09:12 -0800257 /*
258 * Now we know that no one else is looking at the page.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800259 */
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800260 get_page(newpage); /* add cache reference */
Christoph Lameterb20a3502006-03-22 00:09:12 -0800261 if (PageSwapCache(page)) {
262 SetPageSwapCache(newpage);
263 set_page_private(newpage, page_private(page));
264 }
265
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800266 radix_tree_replace_slot(pslot, newpage);
267
Nick Piggine2867812008-07-25 19:45:30 -0700268 page_unfreeze_refs(page, expected_count);
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800269 /*
270 * Drop cache reference from old page.
271 * We know this isn't the last reference.
272 */
Christoph Lameterb20a3502006-03-22 00:09:12 -0800273 __put_page(page);
Nick Piggin7cf9c2c2006-12-06 20:33:44 -0800274
Christoph Lameter0e8c7d02007-04-23 14:41:09 -0700275 /*
276 * If moved to a different zone then also account
277 * the page for that zone. Other VM counters will be
278 * taken care of when we establish references to the
279 * new page and drop references to the old page.
280 *
281 * Note that anonymous pages are accounted for
282 * via NR_FILE_PAGES and NR_ANON_PAGES if they
283 * are mapped to swap space.
284 */
285 __dec_zone_page_state(page, NR_FILE_PAGES);
286 __inc_zone_page_state(newpage, NR_FILE_PAGES);
KOSAKI Motohiro4b021082009-09-21 17:01:33 -0700287 if (PageSwapBacked(page)) {
288 __dec_zone_page_state(page, NR_SHMEM);
289 __inc_zone_page_state(newpage, NR_SHMEM);
290 }
Nick Piggin19fd6232008-07-25 19:45:32 -0700291 spin_unlock_irq(&mapping->tree_lock);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800292
293 return 0;
294}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800295
296/*
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900297 * The expected number of remaining references is the same as that
298 * of migrate_page_move_mapping().
299 */
300int migrate_huge_page_move_mapping(struct address_space *mapping,
301 struct page *newpage, struct page *page)
302{
303 int expected_count;
304 void **pslot;
305
306 if (!mapping) {
307 if (page_count(page) != 1)
308 return -EAGAIN;
309 return 0;
310 }
311
312 spin_lock_irq(&mapping->tree_lock);
313
314 pslot = radix_tree_lookup_slot(&mapping->page_tree,
315 page_index(page));
316
317 expected_count = 2 + page_has_private(page);
318 if (page_count(page) != expected_count ||
319 (struct page *)radix_tree_deref_slot(pslot) != page) {
320 spin_unlock_irq(&mapping->tree_lock);
321 return -EAGAIN;
322 }
323
324 if (!page_freeze_refs(page, expected_count)) {
325 spin_unlock_irq(&mapping->tree_lock);
326 return -EAGAIN;
327 }
328
329 get_page(newpage);
330
331 radix_tree_replace_slot(pslot, newpage);
332
333 page_unfreeze_refs(page, expected_count);
334
335 __put_page(page);
336
337 spin_unlock_irq(&mapping->tree_lock);
338 return 0;
339}
340
341/*
Christoph Lameterb20a3502006-03-22 00:09:12 -0800342 * Copy the page to its new location
343 */
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900344void migrate_page_copy(struct page *newpage, struct page *page)
Christoph Lameterb20a3502006-03-22 00:09:12 -0800345{
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900346 if (PageHuge(page))
347 copy_huge_page(newpage, page);
348 else
349 copy_highpage(newpage, page);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800350
351 if (PageError(page))
352 SetPageError(newpage);
353 if (PageReferenced(page))
354 SetPageReferenced(newpage);
355 if (PageUptodate(page))
356 SetPageUptodate(newpage);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700357 if (TestClearPageActive(page)) {
358 VM_BUG_ON(PageUnevictable(page));
Christoph Lameterb20a3502006-03-22 00:09:12 -0800359 SetPageActive(newpage);
Lee Schermerhorn418b27e2009-12-14 17:59:54 -0800360 } else if (TestClearPageUnevictable(page))
361 SetPageUnevictable(newpage);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800362 if (PageChecked(page))
363 SetPageChecked(newpage);
364 if (PageMappedToDisk(page))
365 SetPageMappedToDisk(newpage);
366
367 if (PageDirty(page)) {
368 clear_page_dirty_for_io(page);
Nick Piggin3a902c52008-04-30 00:55:16 -0700369 /*
370 * Want to mark the page and the radix tree as dirty, and
371 * redo the accounting that clear_page_dirty_for_io undid,
372 * but we can't use set_page_dirty because that function
373 * is actually a signal that all of the page has become dirty.
374 * Wheras only part of our page may be dirty.
375 */
376 __set_page_dirty_nobuffers(newpage);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800377 }
378
Nick Pigginb291f002008-10-18 20:26:44 -0700379 mlock_migrate_page(newpage, page);
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800380 ksm_migrate_page(newpage, page);
Nick Pigginb291f002008-10-18 20:26:44 -0700381
Christoph Lameterb20a3502006-03-22 00:09:12 -0800382 ClearPageSwapCache(page);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800383 ClearPagePrivate(page);
384 set_page_private(page, 0);
385 page->mapping = NULL;
386
387 /*
388 * If any waiters have accumulated on the new page then
389 * wake them up.
390 */
391 if (PageWriteback(newpage))
392 end_page_writeback(newpage);
393}
Christoph Lameterb20a3502006-03-22 00:09:12 -0800394
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700395/************************************************************
396 * Migration functions
397 ***********************************************************/
398
399/* Always fail migration. Used for mappings that are not movable */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700400int fail_migrate_page(struct address_space *mapping,
401 struct page *newpage, struct page *page)
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700402{
403 return -EIO;
404}
405EXPORT_SYMBOL(fail_migrate_page);
406
Christoph Lameterb20a3502006-03-22 00:09:12 -0800407/*
408 * Common logic to directly migrate a single page suitable for
David Howells266cf652009-04-03 16:42:36 +0100409 * pages that do not use PagePrivate/PagePrivate2.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800410 *
411 * Pages are locked upon entry and exit.
412 */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700413int migrate_page(struct address_space *mapping,
414 struct page *newpage, struct page *page)
Christoph Lameterb20a3502006-03-22 00:09:12 -0800415{
416 int rc;
417
418 BUG_ON(PageWriteback(page)); /* Writeback must be complete */
419
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700420 rc = migrate_page_move_mapping(mapping, newpage, page);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800421
422 if (rc)
423 return rc;
424
425 migrate_page_copy(newpage, page);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800426 return 0;
427}
428EXPORT_SYMBOL(migrate_page);
429
David Howells93614012006-09-30 20:45:40 +0200430#ifdef CONFIG_BLOCK
Christoph Lameterb20a3502006-03-22 00:09:12 -0800431/*
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700432 * Migration function for pages with buffers. This function can only be used
433 * if the underlying filesystem guarantees that no other references to "page"
434 * exist.
435 */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700436int buffer_migrate_page(struct address_space *mapping,
437 struct page *newpage, struct page *page)
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700438{
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700439 struct buffer_head *bh, *head;
440 int rc;
441
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700442 if (!page_has_buffers(page))
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700443 return migrate_page(mapping, newpage, page);
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700444
445 head = page_buffers(page);
446
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700447 rc = migrate_page_move_mapping(mapping, newpage, page);
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700448
449 if (rc)
450 return rc;
451
452 bh = head;
453 do {
454 get_bh(bh);
455 lock_buffer(bh);
456 bh = bh->b_this_page;
457
458 } while (bh != head);
459
460 ClearPagePrivate(page);
461 set_page_private(newpage, page_private(page));
462 set_page_private(page, 0);
463 put_page(page);
464 get_page(newpage);
465
466 bh = head;
467 do {
468 set_bh_page(bh, newpage, bh_offset(bh));
469 bh = bh->b_this_page;
470
471 } while (bh != head);
472
473 SetPagePrivate(newpage);
474
475 migrate_page_copy(newpage, page);
476
477 bh = head;
478 do {
479 unlock_buffer(bh);
480 put_bh(bh);
481 bh = bh->b_this_page;
482
483 } while (bh != head);
484
485 return 0;
486}
487EXPORT_SYMBOL(buffer_migrate_page);
David Howells93614012006-09-30 20:45:40 +0200488#endif
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700489
Christoph Lameter04e62a22006-06-23 02:03:38 -0700490/*
491 * Writeback a page to clean the dirty state
492 */
493static int writeout(struct address_space *mapping, struct page *page)
494{
495 struct writeback_control wbc = {
496 .sync_mode = WB_SYNC_NONE,
497 .nr_to_write = 1,
498 .range_start = 0,
499 .range_end = LLONG_MAX,
Christoph Lameter04e62a22006-06-23 02:03:38 -0700500 .for_reclaim = 1
501 };
502 int rc;
503
504 if (!mapping->a_ops->writepage)
505 /* No write method for the address space */
506 return -EINVAL;
507
508 if (!clear_page_dirty_for_io(page))
509 /* Someone else already triggered a write */
510 return -EAGAIN;
511
512 /*
513 * A dirty page may imply that the underlying filesystem has
514 * the page on some queue. So the page must be clean for
515 * migration. Writeout may mean we loose the lock and the
516 * page state is no longer what we checked for earlier.
517 * At this point we know that the migration attempt cannot
518 * be successful.
519 */
520 remove_migration_ptes(page, page);
521
522 rc = mapping->a_ops->writepage(page, &wbc);
Christoph Lameter04e62a22006-06-23 02:03:38 -0700523
524 if (rc != AOP_WRITEPAGE_ACTIVATE)
525 /* unlocked. Relock */
526 lock_page(page);
527
Hugh Dickinsbda85502008-11-19 15:36:36 -0800528 return (rc < 0) ? -EIO : -EAGAIN;
Christoph Lameter04e62a22006-06-23 02:03:38 -0700529}
530
531/*
532 * Default handling if a filesystem does not provide a migration function.
533 */
Christoph Lameter8351a6e2006-06-23 02:03:33 -0700534static int fallback_migrate_page(struct address_space *mapping,
535 struct page *newpage, struct page *page)
536{
Christoph Lameter04e62a22006-06-23 02:03:38 -0700537 if (PageDirty(page))
538 return writeout(mapping, page);
Christoph Lameter8351a6e2006-06-23 02:03:33 -0700539
540 /*
541 * Buffers may be managed in a filesystem specific way.
542 * We must have no buffers or drop them.
543 */
David Howells266cf652009-04-03 16:42:36 +0100544 if (page_has_private(page) &&
Christoph Lameter8351a6e2006-06-23 02:03:33 -0700545 !try_to_release_page(page, GFP_KERNEL))
546 return -EAGAIN;
547
548 return migrate_page(mapping, newpage, page);
549}
550
Christoph Lameter1d8b85c2006-06-23 02:03:28 -0700551/*
Christoph Lametere24f0b82006-06-23 02:03:51 -0700552 * Move a page to a newly allocated page
553 * The page is locked and all ptes have been successfully removed.
554 *
555 * The new page will have replaced the old page if this function
556 * is successful.
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700557 *
558 * Return value:
559 * < 0 - error code
560 * == 0 - success
Christoph Lametere24f0b82006-06-23 02:03:51 -0700561 */
Mel Gorman3fe20112010-05-24 14:32:20 -0700562static int move_to_new_page(struct page *newpage, struct page *page,
563 int remap_swapcache)
Christoph Lametere24f0b82006-06-23 02:03:51 -0700564{
565 struct address_space *mapping;
566 int rc;
567
568 /*
569 * Block others from accessing the page when we get around to
570 * establishing additional references. We are the only one
571 * holding a reference to the new page at this point.
572 */
Nick Piggin529ae9a2008-08-02 12:01:03 +0200573 if (!trylock_page(newpage))
Christoph Lametere24f0b82006-06-23 02:03:51 -0700574 BUG();
575
576 /* Prepare mapping for the new page.*/
577 newpage->index = page->index;
578 newpage->mapping = page->mapping;
Rik van Rielb2e18532008-10-18 20:26:30 -0700579 if (PageSwapBacked(page))
580 SetPageSwapBacked(newpage);
Christoph Lametere24f0b82006-06-23 02:03:51 -0700581
582 mapping = page_mapping(page);
583 if (!mapping)
584 rc = migrate_page(mapping, newpage, page);
585 else if (mapping->a_ops->migratepage)
586 /*
587 * Most pages have a mapping and most filesystems
588 * should provide a migration function. Anonymous
589 * pages are part of swap space which also has its
590 * own migration function. This is the most common
591 * path for page migration.
592 */
593 rc = mapping->a_ops->migratepage(mapping,
594 newpage, page);
595 else
596 rc = fallback_migrate_page(mapping, newpage, page);
597
Mel Gorman3fe20112010-05-24 14:32:20 -0700598 if (rc) {
Christoph Lametere24f0b82006-06-23 02:03:51 -0700599 newpage->mapping = NULL;
Mel Gorman3fe20112010-05-24 14:32:20 -0700600 } else {
601 if (remap_swapcache)
602 remove_migration_ptes(page, newpage);
603 }
Christoph Lametere24f0b82006-06-23 02:03:51 -0700604
605 unlock_page(newpage);
606
607 return rc;
608}
609
610/*
611 * Obtain the lock on page, remove all ptes and migrate the page
612 * to the newly allocated page in newpage.
613 */
Christoph Lameter95a402c2006-06-23 02:03:53 -0700614static int unmap_and_move(new_page_t get_new_page, unsigned long private,
Hugh Dickins62b61f62009-12-14 17:59:33 -0800615 struct page *page, int force, int offlining)
Christoph Lametere24f0b82006-06-23 02:03:51 -0700616{
617 int rc = 0;
Christoph Lameter742755a2006-06-23 02:03:55 -0700618 int *result = NULL;
619 struct page *newpage = get_new_page(page, private, &result);
Mel Gorman3fe20112010-05-24 14:32:20 -0700620 int remap_swapcache = 1;
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700621 int rcu_locked = 0;
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -0800622 int charge = 0;
KAMEZAWA Hiroyukie00e4312009-11-11 14:26:26 -0800623 struct mem_cgroup *mem = NULL;
Mel Gorman3f6c8272010-05-24 14:32:17 -0700624 struct anon_vma *anon_vma = NULL;
Christoph Lameter95a402c2006-06-23 02:03:53 -0700625
626 if (!newpage)
627 return -ENOMEM;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700628
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700629 if (page_count(page) == 1) {
Christoph Lametere24f0b82006-06-23 02:03:51 -0700630 /* page was freed from under us. So we are done. */
Christoph Lameter95a402c2006-06-23 02:03:53 -0700631 goto move_newpage;
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700632 }
Christoph Lametere24f0b82006-06-23 02:03:51 -0700633
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700634 /* prepare cgroup just returns 0 or -ENOMEM */
Christoph Lametere24f0b82006-06-23 02:03:51 -0700635 rc = -EAGAIN;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -0800636
Nick Piggin529ae9a2008-08-02 12:01:03 +0200637 if (!trylock_page(page)) {
Christoph Lametere24f0b82006-06-23 02:03:51 -0700638 if (!force)
Christoph Lameter95a402c2006-06-23 02:03:53 -0700639 goto move_newpage;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700640 lock_page(page);
641 }
642
Hugh Dickins62b61f62009-12-14 17:59:33 -0800643 /*
644 * Only memory hotplug's offline_pages() caller has locked out KSM,
645 * and can safely migrate a KSM page. The other cases have skipped
646 * PageKsm along with PageReserved - but it is only now when we have
647 * the page lock that we can be certain it will not go KSM beneath us
648 * (KSM will not upgrade a page from PageAnon to PageKsm when it sees
649 * its pagecount raised, but only here do we take the page lock which
650 * serializes that).
651 */
652 if (PageKsm(page) && !offlining) {
653 rc = -EBUSY;
654 goto unlock;
655 }
656
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -0800657 /* charge against new page */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -0700658 charge = mem_cgroup_prepare_migration(page, newpage, &mem);
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -0800659 if (charge == -ENOMEM) {
660 rc = -ENOMEM;
661 goto unlock;
662 }
663 BUG_ON(charge);
664
Christoph Lametere24f0b82006-06-23 02:03:51 -0700665 if (PageWriteback(page)) {
666 if (!force)
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -0800667 goto uncharge;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700668 wait_on_page_writeback(page);
669 }
Christoph Lametere24f0b82006-06-23 02:03:51 -0700670 /*
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700671 * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
672 * we cannot notice that anon_vma is freed while we migrates a page.
673 * This rcu_read_lock() delays freeing anon_vma pointer until the end
674 * of migration. File cache pages are no problem because of page_lock()
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700675 * File Caches may use write_page() or lock_page() in migration, then,
676 * just care Anon page here.
Christoph Lametere24f0b82006-06-23 02:03:51 -0700677 */
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700678 if (PageAnon(page)) {
679 rcu_read_lock();
680 rcu_locked = 1;
Mel Gorman67b95092010-05-24 14:32:19 -0700681
Mel Gorman3fe20112010-05-24 14:32:20 -0700682 /* Determine how to safely use anon_vma */
683 if (!page_mapped(page)) {
684 if (!PageSwapCache(page))
685 goto rcu_unlock;
Mel Gorman67b95092010-05-24 14:32:19 -0700686
Mel Gorman3fe20112010-05-24 14:32:20 -0700687 /*
688 * We cannot be sure that the anon_vma of an unmapped
689 * swapcache page is safe to use because we don't
690 * know in advance if the VMA that this page belonged
691 * to still exists. If the VMA and others sharing the
692 * data have been freed, then the anon_vma could
693 * already be invalid.
694 *
695 * To avoid this possibility, swapcache pages get
696 * migrated but are not remapped when migration
697 * completes
698 */
699 remap_swapcache = 0;
700 } else {
701 /*
702 * Take a reference count on the anon_vma if the
703 * page is mapped so that it is guaranteed to
704 * exist when the page is remapped later
705 */
706 anon_vma = page_anon_vma(page);
Rik van Riel76545062010-08-09 17:18:41 -0700707 get_anon_vma(anon_vma);
Mel Gorman3fe20112010-05-24 14:32:20 -0700708 }
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700709 }
Shaohua Li62e1c552008-02-04 22:29:33 -0800710
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700711 /*
Shaohua Li62e1c552008-02-04 22:29:33 -0800712 * Corner case handling:
713 * 1. When a new swap-cache page is read into, it is added to the LRU
714 * and treated as swapcache but it has no rmap yet.
715 * Calling try_to_unmap() against a page->mapping==NULL page will
716 * trigger a BUG. So handle it here.
717 * 2. An orphaned page (see truncate_complete_page) might have
718 * fs-private metadata. The page can be picked up due to memory
719 * offlining. Everywhere else except page reclaim, the page is
720 * invisible to the vm, so the page can not be migrated. So try to
721 * free the metadata, so the page can be freed.
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700722 */
Shaohua Li62e1c552008-02-04 22:29:33 -0800723 if (!page->mapping) {
David Howells266cf652009-04-03 16:42:36 +0100724 if (!PageAnon(page) && page_has_private(page)) {
Shaohua Li62e1c552008-02-04 22:29:33 -0800725 /*
726 * Go direct to try_to_free_buffers() here because
727 * a) that's what try_to_release_page() would do anyway
728 * b) we may be under rcu_read_lock() here, so we can't
729 * use GFP_KERNEL which is what try_to_release_page()
730 * needs to be effective.
731 */
732 try_to_free_buffers(page);
Shaohua Liabfc3482009-09-21 17:01:19 -0700733 goto rcu_unlock;
Shaohua Li62e1c552008-02-04 22:29:33 -0800734 }
Shaohua Liabfc3482009-09-21 17:01:19 -0700735 goto skip_unmap;
Shaohua Li62e1c552008-02-04 22:29:33 -0800736 }
737
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700738 /* Establish migration ptes or remove ptes */
Andi Kleen14fa31b2009-09-16 11:50:10 +0200739 try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700740
Shaohua Liabfc3482009-09-21 17:01:19 -0700741skip_unmap:
Christoph Lametere6a15302006-06-25 05:46:49 -0700742 if (!page_mapped(page))
Mel Gorman3fe20112010-05-24 14:32:20 -0700743 rc = move_to_new_page(newpage, page, remap_swapcache);
Christoph Lametere24f0b82006-06-23 02:03:51 -0700744
Mel Gorman3fe20112010-05-24 14:32:20 -0700745 if (rc && remap_swapcache)
Christoph Lametere24f0b82006-06-23 02:03:51 -0700746 remove_migration_ptes(page, page);
KAMEZAWA Hiroyukidc386d42007-07-26 10:41:07 -0700747rcu_unlock:
Mel Gorman3f6c8272010-05-24 14:32:17 -0700748
749 /* Drop an anon_vma reference if we took one */
Rik van Riel76545062010-08-09 17:18:41 -0700750 if (anon_vma)
751 drop_anon_vma(anon_vma);
Mel Gorman3f6c8272010-05-24 14:32:17 -0700752
KAMEZAWA Hiroyuki989f89c2007-08-30 23:56:21 -0700753 if (rcu_locked)
754 rcu_read_unlock();
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -0800755uncharge:
756 if (!charge)
757 mem_cgroup_end_migration(mem, page, newpage);
Christoph Lametere24f0b82006-06-23 02:03:51 -0700758unlock:
759 unlock_page(page);
Christoph Lameter95a402c2006-06-23 02:03:53 -0700760
Christoph Lametere24f0b82006-06-23 02:03:51 -0700761 if (rc != -EAGAIN) {
Christoph Lameteraaa994b2006-06-23 02:03:52 -0700762 /*
763 * A page that has been migrated has all references
764 * removed and will be freed. A page that has not been
765 * migrated will have kepts its references and be
766 * restored.
767 */
768 list_del(&page->lru);
KOSAKI Motohiroa7312862009-09-21 17:01:37 -0700769 dec_zone_page_state(page, NR_ISOLATED_ANON +
Johannes Weiner6c0b1352009-09-21 17:02:59 -0700770 page_is_file_cache(page));
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700771 putback_lru_page(page);
Christoph Lametere24f0b82006-06-23 02:03:51 -0700772 }
Christoph Lameter95a402c2006-06-23 02:03:53 -0700773
774move_newpage:
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700775
Christoph Lameter95a402c2006-06-23 02:03:53 -0700776 /*
777 * Move the new page to the LRU. If migration was not successful
778 * then this will free the page.
779 */
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700780 putback_lru_page(newpage);
781
Christoph Lameter742755a2006-06-23 02:03:55 -0700782 if (result) {
783 if (rc)
784 *result = rc;
785 else
786 *result = page_to_nid(newpage);
787 }
Christoph Lametere24f0b82006-06-23 02:03:51 -0700788 return rc;
789}
790
791/*
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900792 * Counterpart of unmap_and_move_page() for hugepage migration.
793 *
794 * This function doesn't wait the completion of hugepage I/O
795 * because there is no race between I/O and migration for hugepage.
796 * Note that currently hugepage I/O occurs only in direct I/O
797 * where no lock is held and PG_writeback is irrelevant,
798 * and writeback status of all subpages are counted in the reference
799 * count of the head page (i.e. if all subpages of a 2MB hugepage are
800 * under direct I/O, the reference of the head page is 512 and a bit more.)
801 * This means that when we try to migrate hugepage whose subpages are
802 * doing direct I/O, some references remain after try_to_unmap() and
803 * hugepage migration fails without data corruption.
804 *
805 * There is also no race when direct I/O is issued on the page under migration,
806 * because then pte is replaced with migration swap entry and direct I/O code
807 * will wait in the page fault for migration to complete.
808 */
809static int unmap_and_move_huge_page(new_page_t get_new_page,
810 unsigned long private, struct page *hpage,
811 int force, int offlining)
812{
813 int rc = 0;
814 int *result = NULL;
815 struct page *new_hpage = get_new_page(hpage, private, &result);
816 int rcu_locked = 0;
817 struct anon_vma *anon_vma = NULL;
818
819 if (!new_hpage)
820 return -ENOMEM;
821
822 rc = -EAGAIN;
823
824 if (!trylock_page(hpage)) {
825 if (!force)
826 goto out;
827 lock_page(hpage);
828 }
829
830 if (PageAnon(hpage)) {
831 rcu_read_lock();
832 rcu_locked = 1;
833
834 if (page_mapped(hpage)) {
835 anon_vma = page_anon_vma(hpage);
836 atomic_inc(&anon_vma->external_refcount);
837 }
838 }
839
840 try_to_unmap(hpage, TTU_MIGRATION|TTU_IGNORE_MLOCK|TTU_IGNORE_ACCESS);
841
842 if (!page_mapped(hpage))
843 rc = move_to_new_page(new_hpage, hpage, 1);
844
845 if (rc)
846 remove_migration_ptes(hpage, hpage);
847
848 if (anon_vma && atomic_dec_and_lock(&anon_vma->external_refcount,
849 &anon_vma->lock)) {
850 int empty = list_empty(&anon_vma->head);
851 spin_unlock(&anon_vma->lock);
852 if (empty)
853 anon_vma_free(anon_vma);
854 }
855
856 if (rcu_locked)
857 rcu_read_unlock();
858out:
859 unlock_page(hpage);
860
861 if (rc != -EAGAIN) {
862 list_del(&hpage->lru);
863 put_page(hpage);
864 }
865
866 put_page(new_hpage);
867
868 if (result) {
869 if (rc)
870 *result = rc;
871 else
872 *result = page_to_nid(new_hpage);
873 }
874 return rc;
875}
876
877/*
Christoph Lameterb20a3502006-03-22 00:09:12 -0800878 * migrate_pages
879 *
Christoph Lameter95a402c2006-06-23 02:03:53 -0700880 * The function takes one list of pages to migrate and a function
881 * that determines from the page to be migrated and the private data
882 * the target of the move and allocates the page.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800883 *
884 * The function returns after 10 attempts or if no pages
885 * are movable anymore because to has become empty
Minchan Kimcf608ac2010-10-26 14:21:29 -0700886 * or no retryable pages exist anymore.
887 * Caller should call putback_lru_pages to return pages to the LRU
888 * or free list.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800889 *
Christoph Lameter95a402c2006-06-23 02:03:53 -0700890 * Return: Number of pages not migrated or error code.
Christoph Lameterb20a3502006-03-22 00:09:12 -0800891 */
Christoph Lameter95a402c2006-06-23 02:03:53 -0700892int migrate_pages(struct list_head *from,
Hugh Dickins62b61f62009-12-14 17:59:33 -0800893 new_page_t get_new_page, unsigned long private, int offlining)
Christoph Lameterb20a3502006-03-22 00:09:12 -0800894{
Christoph Lametere24f0b82006-06-23 02:03:51 -0700895 int retry = 1;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800896 int nr_failed = 0;
897 int pass = 0;
898 struct page *page;
899 struct page *page2;
900 int swapwrite = current->flags & PF_SWAPWRITE;
901 int rc;
902
903 if (!swapwrite)
904 current->flags |= PF_SWAPWRITE;
905
Christoph Lametere24f0b82006-06-23 02:03:51 -0700906 for(pass = 0; pass < 10 && retry; pass++) {
907 retry = 0;
Christoph Lameterb20a3502006-03-22 00:09:12 -0800908
Christoph Lametere24f0b82006-06-23 02:03:51 -0700909 list_for_each_entry_safe(page, page2, from, lru) {
Christoph Lametere24f0b82006-06-23 02:03:51 -0700910 cond_resched();
Christoph Lameterb20a3502006-03-22 00:09:12 -0800911
Christoph Lameter95a402c2006-06-23 02:03:53 -0700912 rc = unmap_and_move(get_new_page, private,
Hugh Dickins62b61f62009-12-14 17:59:33 -0800913 page, pass > 2, offlining);
Christoph Lameterb20a3502006-03-22 00:09:12 -0800914
Christoph Lametere24f0b82006-06-23 02:03:51 -0700915 switch(rc) {
Christoph Lameter95a402c2006-06-23 02:03:53 -0700916 case -ENOMEM:
917 goto out;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700918 case -EAGAIN:
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700919 retry++;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700920 break;
921 case 0:
Christoph Lametere24f0b82006-06-23 02:03:51 -0700922 break;
923 default:
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700924 /* Permanent failure */
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700925 nr_failed++;
Christoph Lametere24f0b82006-06-23 02:03:51 -0700926 break;
Christoph Lameter2d1db3b2006-06-23 02:03:33 -0700927 }
Christoph Lameterb20a3502006-03-22 00:09:12 -0800928 }
929 }
Christoph Lameter95a402c2006-06-23 02:03:53 -0700930 rc = 0;
931out:
Christoph Lameterb20a3502006-03-22 00:09:12 -0800932 if (!swapwrite)
933 current->flags &= ~PF_SWAPWRITE;
934
Christoph Lameter95a402c2006-06-23 02:03:53 -0700935 if (rc)
936 return rc;
937
Christoph Lameterb20a3502006-03-22 00:09:12 -0800938 return nr_failed + retry;
939}
940
Naoya Horiguchi290408d2010-09-08 10:19:35 +0900941int migrate_huge_pages(struct list_head *from,
942 new_page_t get_new_page, unsigned long private, int offlining)
943{
944 int retry = 1;
945 int nr_failed = 0;
946 int pass = 0;
947 struct page *page;
948 struct page *page2;
949 int rc;
950
951 for (pass = 0; pass < 10 && retry; pass++) {
952 retry = 0;
953
954 list_for_each_entry_safe(page, page2, from, lru) {
955 cond_resched();
956
957 rc = unmap_and_move_huge_page(get_new_page,
958 private, page, pass > 2, offlining);
959
960 switch(rc) {
961 case -ENOMEM:
962 goto out;
963 case -EAGAIN:
964 retry++;
965 break;
966 case 0:
967 break;
968 default:
969 /* Permanent failure */
970 nr_failed++;
971 break;
972 }
973 }
974 }
975 rc = 0;
976out:
977
978 list_for_each_entry_safe(page, page2, from, lru)
979 put_page(page);
980
981 if (rc)
982 return rc;
983
984 return nr_failed + retry;
985}
986
Christoph Lameter742755a2006-06-23 02:03:55 -0700987#ifdef CONFIG_NUMA
988/*
989 * Move a list of individual pages
990 */
991struct page_to_node {
992 unsigned long addr;
993 struct page *page;
994 int node;
995 int status;
996};
997
998static struct page *new_page_node(struct page *p, unsigned long private,
999 int **result)
1000{
1001 struct page_to_node *pm = (struct page_to_node *)private;
1002
1003 while (pm->node != MAX_NUMNODES && pm->page != p)
1004 pm++;
1005
1006 if (pm->node == MAX_NUMNODES)
1007 return NULL;
1008
1009 *result = &pm->status;
1010
Mel Gorman6484eb32009-06-16 15:31:54 -07001011 return alloc_pages_exact_node(pm->node,
Mel Gorman769848c2007-07-17 04:03:05 -07001012 GFP_HIGHUSER_MOVABLE | GFP_THISNODE, 0);
Christoph Lameter742755a2006-06-23 02:03:55 -07001013}
1014
1015/*
1016 * Move a set of pages as indicated in the pm array. The addr
1017 * field must be set to the virtual address of the page to be moved
1018 * and the node number must contain a valid target node.
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001019 * The pm array ends with node = MAX_NUMNODES.
Christoph Lameter742755a2006-06-23 02:03:55 -07001020 */
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001021static int do_move_page_to_node_array(struct mm_struct *mm,
1022 struct page_to_node *pm,
1023 int migrate_all)
Christoph Lameter742755a2006-06-23 02:03:55 -07001024{
1025 int err;
1026 struct page_to_node *pp;
1027 LIST_HEAD(pagelist);
1028
1029 down_read(&mm->mmap_sem);
1030
1031 /*
1032 * Build a list of pages to migrate
1033 */
Christoph Lameter742755a2006-06-23 02:03:55 -07001034 for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
1035 struct vm_area_struct *vma;
1036 struct page *page;
1037
Christoph Lameter742755a2006-06-23 02:03:55 -07001038 err = -EFAULT;
1039 vma = find_vma(mm, pp->addr);
Christoph Lameter0dc952d2007-03-05 00:30:33 -08001040 if (!vma || !vma_migratable(vma))
Christoph Lameter742755a2006-06-23 02:03:55 -07001041 goto set_status;
1042
1043 page = follow_page(vma, pp->addr, FOLL_GET);
Linus Torvalds89f5b7d2008-06-20 11:18:25 -07001044
1045 err = PTR_ERR(page);
1046 if (IS_ERR(page))
1047 goto set_status;
1048
Christoph Lameter742755a2006-06-23 02:03:55 -07001049 err = -ENOENT;
1050 if (!page)
1051 goto set_status;
1052
Hugh Dickins62b61f62009-12-14 17:59:33 -08001053 /* Use PageReserved to check for zero page */
1054 if (PageReserved(page) || PageKsm(page))
Christoph Lameter742755a2006-06-23 02:03:55 -07001055 goto put_and_set;
1056
1057 pp->page = page;
1058 err = page_to_nid(page);
1059
1060 if (err == pp->node)
1061 /*
1062 * Node already in the right place
1063 */
1064 goto put_and_set;
1065
1066 err = -EACCES;
1067 if (page_mapcount(page) > 1 &&
1068 !migrate_all)
1069 goto put_and_set;
1070
Nick Piggin62695a82008-10-18 20:26:09 -07001071 err = isolate_lru_page(page);
KOSAKI Motohiro6d9c2852009-12-14 17:58:11 -08001072 if (!err) {
Nick Piggin62695a82008-10-18 20:26:09 -07001073 list_add_tail(&page->lru, &pagelist);
KOSAKI Motohiro6d9c2852009-12-14 17:58:11 -08001074 inc_zone_page_state(page, NR_ISOLATED_ANON +
1075 page_is_file_cache(page));
1076 }
Christoph Lameter742755a2006-06-23 02:03:55 -07001077put_and_set:
1078 /*
1079 * Either remove the duplicate refcount from
1080 * isolate_lru_page() or drop the page ref if it was
1081 * not isolated.
1082 */
1083 put_page(page);
1084set_status:
1085 pp->status = err;
1086 }
1087
Brice Gogline78bbfa2008-10-18 20:27:15 -07001088 err = 0;
Minchan Kimcf608ac2010-10-26 14:21:29 -07001089 if (!list_empty(&pagelist)) {
Christoph Lameter742755a2006-06-23 02:03:55 -07001090 err = migrate_pages(&pagelist, new_page_node,
Hugh Dickins62b61f62009-12-14 17:59:33 -08001091 (unsigned long)pm, 0);
Minchan Kimcf608ac2010-10-26 14:21:29 -07001092 if (err)
1093 putback_lru_pages(&pagelist);
1094 }
Christoph Lameter742755a2006-06-23 02:03:55 -07001095
1096 up_read(&mm->mmap_sem);
1097 return err;
1098}
1099
1100/*
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001101 * Migrate an array of page address onto an array of nodes and fill
1102 * the corresponding array of status.
1103 */
1104static int do_pages_move(struct mm_struct *mm, struct task_struct *task,
1105 unsigned long nr_pages,
1106 const void __user * __user *pages,
1107 const int __user *nodes,
1108 int __user *status, int flags)
1109{
Brice Goglin3140a222009-01-06 14:38:57 -08001110 struct page_to_node *pm;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001111 nodemask_t task_nodes;
Brice Goglin3140a222009-01-06 14:38:57 -08001112 unsigned long chunk_nr_pages;
1113 unsigned long chunk_start;
1114 int err;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001115
1116 task_nodes = cpuset_mems_allowed(task);
1117
Brice Goglin3140a222009-01-06 14:38:57 -08001118 err = -ENOMEM;
1119 pm = (struct page_to_node *)__get_free_page(GFP_KERNEL);
1120 if (!pm)
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001121 goto out;
Brice Goglin35282a22009-06-16 15:32:43 -07001122
1123 migrate_prep();
1124
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001125 /*
Brice Goglin3140a222009-01-06 14:38:57 -08001126 * Store a chunk of page_to_node array in a page,
1127 * but keep the last one as a marker
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001128 */
Brice Goglin3140a222009-01-06 14:38:57 -08001129 chunk_nr_pages = (PAGE_SIZE / sizeof(struct page_to_node)) - 1;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001130
Brice Goglin3140a222009-01-06 14:38:57 -08001131 for (chunk_start = 0;
1132 chunk_start < nr_pages;
1133 chunk_start += chunk_nr_pages) {
1134 int j;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001135
Brice Goglin3140a222009-01-06 14:38:57 -08001136 if (chunk_start + chunk_nr_pages > nr_pages)
1137 chunk_nr_pages = nr_pages - chunk_start;
1138
1139 /* fill the chunk pm with addrs and nodes from user-space */
1140 for (j = 0; j < chunk_nr_pages; j++) {
1141 const void __user *p;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001142 int node;
1143
Brice Goglin3140a222009-01-06 14:38:57 -08001144 err = -EFAULT;
1145 if (get_user(p, pages + j + chunk_start))
1146 goto out_pm;
1147 pm[j].addr = (unsigned long) p;
1148
1149 if (get_user(node, nodes + j + chunk_start))
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001150 goto out_pm;
1151
1152 err = -ENODEV;
Linus Torvalds6f5a55f2010-02-05 16:16:50 -08001153 if (node < 0 || node >= MAX_NUMNODES)
1154 goto out_pm;
1155
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001156 if (!node_state(node, N_HIGH_MEMORY))
1157 goto out_pm;
1158
1159 err = -EACCES;
1160 if (!node_isset(node, task_nodes))
1161 goto out_pm;
1162
Brice Goglin3140a222009-01-06 14:38:57 -08001163 pm[j].node = node;
1164 }
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001165
Brice Goglin3140a222009-01-06 14:38:57 -08001166 /* End marker for this chunk */
1167 pm[chunk_nr_pages].node = MAX_NUMNODES;
1168
1169 /* Migrate this chunk */
1170 err = do_move_page_to_node_array(mm, pm,
1171 flags & MPOL_MF_MOVE_ALL);
1172 if (err < 0)
1173 goto out_pm;
1174
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001175 /* Return status information */
Brice Goglin3140a222009-01-06 14:38:57 -08001176 for (j = 0; j < chunk_nr_pages; j++)
1177 if (put_user(pm[j].status, status + j + chunk_start)) {
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001178 err = -EFAULT;
Brice Goglin3140a222009-01-06 14:38:57 -08001179 goto out_pm;
1180 }
1181 }
1182 err = 0;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001183
1184out_pm:
Brice Goglin3140a222009-01-06 14:38:57 -08001185 free_page((unsigned long)pm);
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001186out:
1187 return err;
1188}
1189
1190/*
Brice Goglin2f007e72008-10-18 20:27:16 -07001191 * Determine the nodes of an array of pages and store it in an array of status.
Christoph Lameter742755a2006-06-23 02:03:55 -07001192 */
Brice Goglin80bba122008-12-09 13:14:23 -08001193static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
1194 const void __user **pages, int *status)
Christoph Lameter742755a2006-06-23 02:03:55 -07001195{
Brice Goglin2f007e72008-10-18 20:27:16 -07001196 unsigned long i;
Brice Goglin2f007e72008-10-18 20:27:16 -07001197
Christoph Lameter742755a2006-06-23 02:03:55 -07001198 down_read(&mm->mmap_sem);
1199
Brice Goglin2f007e72008-10-18 20:27:16 -07001200 for (i = 0; i < nr_pages; i++) {
Brice Goglin80bba122008-12-09 13:14:23 -08001201 unsigned long addr = (unsigned long)(*pages);
Christoph Lameter742755a2006-06-23 02:03:55 -07001202 struct vm_area_struct *vma;
1203 struct page *page;
KOSAKI Motohiroc095adb2008-12-16 16:06:43 +09001204 int err = -EFAULT;
Brice Goglin2f007e72008-10-18 20:27:16 -07001205
1206 vma = find_vma(mm, addr);
Christoph Lameter742755a2006-06-23 02:03:55 -07001207 if (!vma)
1208 goto set_status;
1209
Brice Goglin2f007e72008-10-18 20:27:16 -07001210 page = follow_page(vma, addr, 0);
Linus Torvalds89f5b7d2008-06-20 11:18:25 -07001211
1212 err = PTR_ERR(page);
1213 if (IS_ERR(page))
1214 goto set_status;
1215
Christoph Lameter742755a2006-06-23 02:03:55 -07001216 err = -ENOENT;
1217 /* Use PageReserved to check for zero page */
Hugh Dickins62b61f62009-12-14 17:59:33 -08001218 if (!page || PageReserved(page) || PageKsm(page))
Christoph Lameter742755a2006-06-23 02:03:55 -07001219 goto set_status;
1220
1221 err = page_to_nid(page);
1222set_status:
Brice Goglin80bba122008-12-09 13:14:23 -08001223 *status = err;
1224
1225 pages++;
1226 status++;
1227 }
1228
1229 up_read(&mm->mmap_sem);
1230}
1231
1232/*
1233 * Determine the nodes of a user array of pages and store it in
1234 * a user array of status.
1235 */
1236static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
1237 const void __user * __user *pages,
1238 int __user *status)
1239{
1240#define DO_PAGES_STAT_CHUNK_NR 16
1241 const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
1242 int chunk_status[DO_PAGES_STAT_CHUNK_NR];
Brice Goglin80bba122008-12-09 13:14:23 -08001243
H. Peter Anvin87b8d1a2010-02-18 16:13:40 -08001244 while (nr_pages) {
1245 unsigned long chunk_nr;
Brice Goglin80bba122008-12-09 13:14:23 -08001246
H. Peter Anvin87b8d1a2010-02-18 16:13:40 -08001247 chunk_nr = nr_pages;
1248 if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
1249 chunk_nr = DO_PAGES_STAT_CHUNK_NR;
1250
1251 if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
1252 break;
Brice Goglin80bba122008-12-09 13:14:23 -08001253
1254 do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
1255
H. Peter Anvin87b8d1a2010-02-18 16:13:40 -08001256 if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
1257 break;
Christoph Lameter742755a2006-06-23 02:03:55 -07001258
H. Peter Anvin87b8d1a2010-02-18 16:13:40 -08001259 pages += chunk_nr;
1260 status += chunk_nr;
1261 nr_pages -= chunk_nr;
1262 }
1263 return nr_pages ? -EFAULT : 0;
Christoph Lameter742755a2006-06-23 02:03:55 -07001264}
1265
1266/*
1267 * Move a list of pages in the address space of the currently executing
1268 * process.
1269 */
Heiko Carstens938bb9f2009-01-14 14:14:30 +01001270SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
1271 const void __user * __user *, pages,
1272 const int __user *, nodes,
1273 int __user *, status, int, flags)
Christoph Lameter742755a2006-06-23 02:03:55 -07001274{
David Howellsc69e8d92008-11-14 10:39:19 +11001275 const struct cred *cred = current_cred(), *tcred;
Christoph Lameter742755a2006-06-23 02:03:55 -07001276 struct task_struct *task;
Christoph Lameter742755a2006-06-23 02:03:55 -07001277 struct mm_struct *mm;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001278 int err;
Christoph Lameter742755a2006-06-23 02:03:55 -07001279
1280 /* Check flags */
1281 if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
1282 return -EINVAL;
1283
1284 if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
1285 return -EPERM;
1286
1287 /* Find the mm_struct */
1288 read_lock(&tasklist_lock);
Pavel Emelyanov228ebcb2007-10-18 23:40:16 -07001289 task = pid ? find_task_by_vpid(pid) : current;
Christoph Lameter742755a2006-06-23 02:03:55 -07001290 if (!task) {
1291 read_unlock(&tasklist_lock);
1292 return -ESRCH;
1293 }
1294 mm = get_task_mm(task);
1295 read_unlock(&tasklist_lock);
1296
1297 if (!mm)
1298 return -EINVAL;
1299
1300 /*
1301 * Check if this process has the right to modify the specified
1302 * process. The right exists if the process has administrative
1303 * capabilities, superuser privileges or the same
1304 * userid as the target process.
1305 */
David Howellsc69e8d92008-11-14 10:39:19 +11001306 rcu_read_lock();
1307 tcred = __task_cred(task);
David Howellsb6dff3e2008-11-14 10:39:16 +11001308 if (cred->euid != tcred->suid && cred->euid != tcred->uid &&
1309 cred->uid != tcred->suid && cred->uid != tcred->uid &&
Christoph Lameter742755a2006-06-23 02:03:55 -07001310 !capable(CAP_SYS_NICE)) {
David Howellsc69e8d92008-11-14 10:39:19 +11001311 rcu_read_unlock();
Christoph Lameter742755a2006-06-23 02:03:55 -07001312 err = -EPERM;
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001313 goto out;
Christoph Lameter742755a2006-06-23 02:03:55 -07001314 }
David Howellsc69e8d92008-11-14 10:39:19 +11001315 rcu_read_unlock();
Christoph Lameter742755a2006-06-23 02:03:55 -07001316
David Quigley86c3a762006-06-23 02:04:02 -07001317 err = security_task_movememory(task);
1318 if (err)
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001319 goto out;
David Quigley86c3a762006-06-23 02:04:02 -07001320
Brice Goglin5e9a0f02008-10-18 20:27:17 -07001321 if (nodes) {
1322 err = do_pages_move(mm, task, nr_pages, pages, nodes, status,
1323 flags);
1324 } else {
Brice Goglin2f007e72008-10-18 20:27:16 -07001325 err = do_pages_stat(mm, nr_pages, pages, status);
Brice Goglin2f007e72008-10-18 20:27:16 -07001326 }
David Quigley86c3a762006-06-23 02:04:02 -07001327
Christoph Lameter742755a2006-06-23 02:03:55 -07001328out:
Christoph Lameter742755a2006-06-23 02:03:55 -07001329 mmput(mm);
1330 return err;
1331}
Christoph Lameter742755a2006-06-23 02:03:55 -07001332
Christoph Lameter7b2259b2006-06-25 05:46:48 -07001333/*
1334 * Call migration functions in the vma_ops that may prepare
1335 * memory in a vm for migration. migration functions may perform
1336 * the migration for vmas that do not have an underlying page struct.
1337 */
1338int migrate_vmas(struct mm_struct *mm, const nodemask_t *to,
1339 const nodemask_t *from, unsigned long flags)
1340{
1341 struct vm_area_struct *vma;
1342 int err = 0;
1343
Daisuke Nishimura1001c9f2009-02-11 13:04:18 -08001344 for (vma = mm->mmap; vma && !err; vma = vma->vm_next) {
Christoph Lameter7b2259b2006-06-25 05:46:48 -07001345 if (vma->vm_ops && vma->vm_ops->migrate) {
1346 err = vma->vm_ops->migrate(vma, to, from, flags);
1347 if (err)
1348 break;
1349 }
1350 }
1351 return err;
1352}
Gerald Schaefer83d16742008-07-23 21:28:22 -07001353#endif