KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/mm/page_isolation.c |
| 3 | */ |
| 4 | |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 5 | #include <linux/mm.h> |
| 6 | #include <linux/page-isolation.h> |
| 7 | #include <linux/pageblock-flags.h> |
| 8 | #include "internal.h" |
| 9 | |
| 10 | static inline struct page * |
| 11 | __first_valid_page(unsigned long pfn, unsigned long nr_pages) |
| 12 | { |
| 13 | int i; |
| 14 | for (i = 0; i < nr_pages; i++) |
| 15 | if (pfn_valid_within(pfn + i)) |
| 16 | break; |
| 17 | if (unlikely(i == nr_pages)) |
| 18 | return NULL; |
| 19 | return pfn_to_page(pfn + i); |
| 20 | } |
| 21 | |
| 22 | /* |
| 23 | * start_isolate_page_range() -- make page-allocation-type of range of pages |
| 24 | * to be MIGRATE_ISOLATE. |
| 25 | * @start_pfn: The lower PFN of the range to be isolated. |
| 26 | * @end_pfn: The upper PFN of the range to be isolated. |
Michal Nazarewicz | c80cd92 | 2012-04-03 15:06:15 +0200 | [diff] [blame^] | 27 | * @migratetype: migrate type to set in error recovery. |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 28 | * |
| 29 | * Making page-allocation-type to be MIGRATE_ISOLATE means free pages in |
| 30 | * the range will never be allocated. Any free pages and pages freed in the |
| 31 | * future will not be allocated again. |
| 32 | * |
| 33 | * start_pfn/end_pfn must be aligned to pageblock_order. |
| 34 | * Returns 0 on success and -EBUSY if any part of range cannot be isolated. |
| 35 | */ |
Michal Nazarewicz | c80cd92 | 2012-04-03 15:06:15 +0200 | [diff] [blame^] | 36 | int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, |
| 37 | unsigned migratetype) |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 38 | { |
| 39 | unsigned long pfn; |
| 40 | unsigned long undo_pfn; |
| 41 | struct page *page; |
| 42 | |
| 43 | BUG_ON((start_pfn) & (pageblock_nr_pages - 1)); |
| 44 | BUG_ON((end_pfn) & (pageblock_nr_pages - 1)); |
| 45 | |
| 46 | for (pfn = start_pfn; |
| 47 | pfn < end_pfn; |
| 48 | pfn += pageblock_nr_pages) { |
| 49 | page = __first_valid_page(pfn, pageblock_nr_pages); |
| 50 | if (page && set_migratetype_isolate(page)) { |
| 51 | undo_pfn = pfn; |
| 52 | goto undo; |
| 53 | } |
| 54 | } |
| 55 | return 0; |
| 56 | undo: |
| 57 | for (pfn = start_pfn; |
KAMEZAWA Hiroyuki | dbc0e4c | 2007-11-14 16:59:12 -0800 | [diff] [blame] | 58 | pfn < undo_pfn; |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 59 | pfn += pageblock_nr_pages) |
Michal Nazarewicz | c80cd92 | 2012-04-03 15:06:15 +0200 | [diff] [blame^] | 60 | unset_migratetype_isolate(pfn_to_page(pfn), migratetype); |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 61 | |
| 62 | return -EBUSY; |
| 63 | } |
| 64 | |
| 65 | /* |
| 66 | * Make isolated pages available again. |
| 67 | */ |
Michal Nazarewicz | c80cd92 | 2012-04-03 15:06:15 +0200 | [diff] [blame^] | 68 | int undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn, |
| 69 | unsigned migratetype) |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 70 | { |
| 71 | unsigned long pfn; |
| 72 | struct page *page; |
| 73 | BUG_ON((start_pfn) & (pageblock_nr_pages - 1)); |
| 74 | BUG_ON((end_pfn) & (pageblock_nr_pages - 1)); |
| 75 | for (pfn = start_pfn; |
| 76 | pfn < end_pfn; |
| 77 | pfn += pageblock_nr_pages) { |
| 78 | page = __first_valid_page(pfn, pageblock_nr_pages); |
KAMEZAWA Hiroyuki | dbc0e4c | 2007-11-14 16:59:12 -0800 | [diff] [blame] | 79 | if (!page || get_pageblock_migratetype(page) != MIGRATE_ISOLATE) |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 80 | continue; |
Michal Nazarewicz | c80cd92 | 2012-04-03 15:06:15 +0200 | [diff] [blame^] | 81 | unset_migratetype_isolate(page, migratetype); |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 82 | } |
| 83 | return 0; |
| 84 | } |
| 85 | /* |
| 86 | * Test all pages in the range is free(means isolated) or not. |
| 87 | * all pages in [start_pfn...end_pfn) must be in the same zone. |
| 88 | * zone->lock must be held before call this. |
| 89 | * |
Michal Nazarewicz | c80cd92 | 2012-04-03 15:06:15 +0200 | [diff] [blame^] | 90 | * Returns 1 if all pages in the range are isolated. |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 91 | */ |
| 92 | static int |
| 93 | __test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn) |
| 94 | { |
| 95 | struct page *page; |
| 96 | |
| 97 | while (pfn < end_pfn) { |
| 98 | if (!pfn_valid_within(pfn)) { |
| 99 | pfn++; |
| 100 | continue; |
| 101 | } |
| 102 | page = pfn_to_page(pfn); |
| 103 | if (PageBuddy(page)) |
| 104 | pfn += 1 << page_order(page); |
| 105 | else if (page_count(page) == 0 && |
| 106 | page_private(page) == MIGRATE_ISOLATE) |
| 107 | pfn += 1; |
| 108 | else |
| 109 | break; |
| 110 | } |
| 111 | if (pfn < end_pfn) |
| 112 | return 0; |
| 113 | return 1; |
| 114 | } |
| 115 | |
| 116 | int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn) |
| 117 | { |
Gerald Schaefer | 6c1b7f6 | 2008-10-02 14:50:16 -0700 | [diff] [blame] | 118 | unsigned long pfn, flags; |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 119 | struct page *page; |
Gerald Schaefer | 6c1b7f6 | 2008-10-02 14:50:16 -0700 | [diff] [blame] | 120 | struct zone *zone; |
| 121 | int ret; |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 122 | |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 123 | /* |
| 124 | * Note: pageblock_nr_page != MAX_ORDER. Then, chunks of free page |
| 125 | * is not aligned to pageblock_nr_pages. |
| 126 | * Then we just check pagetype fist. |
| 127 | */ |
| 128 | for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) { |
| 129 | page = __first_valid_page(pfn, pageblock_nr_pages); |
KAMEZAWA Hiroyuki | dbc0e4c | 2007-11-14 16:59:12 -0800 | [diff] [blame] | 130 | if (page && get_pageblock_migratetype(page) != MIGRATE_ISOLATE) |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 131 | break; |
| 132 | } |
Gerald Schaefer | a70dcb9 | 2008-11-06 12:53:36 -0800 | [diff] [blame] | 133 | page = __first_valid_page(start_pfn, end_pfn - start_pfn); |
| 134 | if ((pfn < end_pfn) || !page) |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 135 | return -EBUSY; |
| 136 | /* Check all pages are free or Marked as ISOLATED */ |
Gerald Schaefer | a70dcb9 | 2008-11-06 12:53:36 -0800 | [diff] [blame] | 137 | zone = page_zone(page); |
Gerald Schaefer | 6c1b7f6 | 2008-10-02 14:50:16 -0700 | [diff] [blame] | 138 | spin_lock_irqsave(&zone->lock, flags); |
| 139 | ret = __test_page_isolated_in_pageblock(start_pfn, end_pfn); |
| 140 | spin_unlock_irqrestore(&zone->lock, flags); |
| 141 | return ret ? 0 : -EBUSY; |
KAMEZAWA Hiroyuki | a5d76b5 | 2007-10-16 01:26:11 -0700 | [diff] [blame] | 142 | } |