blob: 4568fd58f70a02c472e525528db58eeb20c06ebf [file] [log] [blame]
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07001/*
2 * linux/mm/page_isolation.c
3 */
4
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07005#include <linux/mm.h>
6#include <linux/page-isolation.h>
7#include <linux/pageblock-flags.h>
Minchan Kimee6f5092012-07-31 16:43:50 -07008#include <linux/memory.h>
Naoya Horiguchic8721bb2013-09-11 14:22:09 -07009#include <linux/hugetlb.h>
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -070010#include "internal.h"
11
Naoya Horiguchic5b4e1b2015-09-08 15:02:09 -070012static int set_migratetype_isolate(struct page *page,
13 bool skip_hwpoisoned_pages)
Minchan Kimee6f5092012-07-31 16:43:50 -070014{
15 struct zone *zone;
16 unsigned long flags, pfn;
17 struct memory_isolate_notify arg;
18 int notifier_ret;
19 int ret = -EBUSY;
20
21 zone = page_zone(page);
22
23 spin_lock_irqsave(&zone->lock, flags);
24
25 pfn = page_to_pfn(page);
26 arg.start_pfn = pfn;
27 arg.nr_pages = pageblock_nr_pages;
28 arg.pages_found = 0;
29
30 /*
31 * It may be possible to isolate a pageblock even if the
32 * migratetype is not MIGRATE_MOVABLE. The memory isolation
33 * notifier chain is used by balloon drivers to return the
34 * number of pages in a range that are held by the balloon
35 * driver to shrink memory. If all the pages are accounted for
36 * by balloons, are free, or on the LRU, isolation can continue.
37 * Later, for example, when memory hotplug notifier runs, these
38 * pages reported as "can be isolated" should be isolated(freed)
39 * by the balloon driver through the memory notifier chain.
40 */
41 notifier_ret = memory_isolate_notify(MEM_ISOLATE_COUNT, &arg);
42 notifier_ret = notifier_to_errno(notifier_ret);
43 if (notifier_ret)
44 goto out;
45 /*
46 * FIXME: Now, memory hotplug doesn't call shrink_slab() by itself.
47 * We just check MOVABLE pages.
48 */
Wen Congyangb023f462012-12-11 16:00:45 -080049 if (!has_unmovable_pages(zone, page, arg.pages_found,
50 skip_hwpoisoned_pages))
Minchan Kimee6f5092012-07-31 16:43:50 -070051 ret = 0;
52
53 /*
54 * immobile means "not-on-lru" paes. If immobile is larger than
55 * removable-by-driver pages reported by notifier, we'll fail.
56 */
57
58out:
59 if (!ret) {
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -070060 unsigned long nr_pages;
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -070061 int migratetype = get_pageblock_migratetype(page);
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -070062
Bartlomiej Zolnierkiewicza4584312013-01-04 15:35:08 -080063 set_pageblock_migratetype(page, MIGRATE_ISOLATE);
Joonsoo Kimad53f922014-11-13 15:19:11 -080064 zone->nr_isolate_pageblock++;
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -070065 nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE);
66
Bartlomiej Zolnierkiewiczd1ce7492012-10-08 16:32:02 -070067 __mod_zone_freepage_state(zone, -nr_pages, migratetype);
Minchan Kimee6f5092012-07-31 16:43:50 -070068 }
69
70 spin_unlock_irqrestore(&zone->lock, flags);
71 if (!ret)
Vlastimil Babkaec25af82014-12-10 15:43:04 -080072 drain_all_pages(zone);
Minchan Kimee6f5092012-07-31 16:43:50 -070073 return ret;
74}
75
Naoya Horiguchic5b4e1b2015-09-08 15:02:09 -070076static void unset_migratetype_isolate(struct page *page, unsigned migratetype)
Minchan Kimee6f5092012-07-31 16:43:50 -070077{
78 struct zone *zone;
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -070079 unsigned long flags, nr_pages;
Joonsoo Kim3c605092014-11-13 15:19:21 -080080 struct page *isolated_page = NULL;
81 unsigned int order;
82 unsigned long page_idx, buddy_idx;
83 struct page *buddy;
Bartlomiej Zolnierkiewicz2139cbe2012-10-08 16:32:00 -070084
Minchan Kimee6f5092012-07-31 16:43:50 -070085 zone = page_zone(page);
86 spin_lock_irqsave(&zone->lock, flags);
87 if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
88 goto out;
Joonsoo Kim3c605092014-11-13 15:19:21 -080089
90 /*
91 * Because freepage with more than pageblock_order on isolated
92 * pageblock is restricted to merge due to freepage counting problem,
93 * it is possible that there is free buddy page.
94 * move_freepages_block() doesn't care of merge so we need other
95 * approach in order to merge them. Isolation and free will make
96 * these pages to be merged.
97 */
98 if (PageBuddy(page)) {
99 order = page_order(page);
100 if (order >= pageblock_order) {
101 page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
102 buddy_idx = __find_buddy_index(page_idx, order);
103 buddy = page + (buddy_idx - page_idx);
104
Hui Zhu1ae70132015-05-14 15:17:04 -0700105 if (pfn_valid_within(page_to_pfn(buddy)) &&
106 !is_migrate_isolate_page(buddy)) {
Joonsoo Kim3c605092014-11-13 15:19:21 -0800107 __isolate_free_page(page, order);
Laura Abbottcfa86942015-03-25 15:55:26 -0700108 kernel_map_pages(page, (1 << order), 1);
Joonsoo Kim3c605092014-11-13 15:19:21 -0800109 set_page_refcounted(page);
110 isolated_page = page;
111 }
112 }
113 }
114
115 /*
116 * If we isolate freepage with more than pageblock_order, there
117 * should be no freepage in the range, so we could avoid costly
118 * pageblock scanning for freepage moving.
119 */
120 if (!isolated_page) {
121 nr_pages = move_freepages_block(zone, page, migratetype);
122 __mod_zone_freepage_state(zone, nr_pages, migratetype);
123 }
Bartlomiej Zolnierkiewicza4584312013-01-04 15:35:08 -0800124 set_pageblock_migratetype(page, migratetype);
Joonsoo Kimad53f922014-11-13 15:19:11 -0800125 zone->nr_isolate_pageblock--;
Minchan Kimee6f5092012-07-31 16:43:50 -0700126out:
127 spin_unlock_irqrestore(&zone->lock, flags);
Joonsoo Kim3c605092014-11-13 15:19:21 -0800128 if (isolated_page)
129 __free_pages(isolated_page, order);
Minchan Kimee6f5092012-07-31 16:43:50 -0700130}
131
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700132static inline struct page *
133__first_valid_page(unsigned long pfn, unsigned long nr_pages)
134{
135 int i;
136 for (i = 0; i < nr_pages; i++)
137 if (pfn_valid_within(pfn + i))
138 break;
139 if (unlikely(i == nr_pages))
140 return NULL;
141 return pfn_to_page(pfn + i);
142}
143
144/*
145 * start_isolate_page_range() -- make page-allocation-type of range of pages
146 * to be MIGRATE_ISOLATE.
147 * @start_pfn: The lower PFN of the range to be isolated.
148 * @end_pfn: The upper PFN of the range to be isolated.
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +0200149 * @migratetype: migrate type to set in error recovery.
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700150 *
151 * Making page-allocation-type to be MIGRATE_ISOLATE means free pages in
152 * the range will never be allocated. Any free pages and pages freed in the
153 * future will not be allocated again.
154 *
155 * start_pfn/end_pfn must be aligned to pageblock_order.
156 * Returns 0 on success and -EBUSY if any part of range cannot be isolated.
157 */
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +0200158int start_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
Wen Congyangb023f462012-12-11 16:00:45 -0800159 unsigned migratetype, bool skip_hwpoisoned_pages)
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700160{
161 unsigned long pfn;
162 unsigned long undo_pfn;
163 struct page *page;
164
165 BUG_ON((start_pfn) & (pageblock_nr_pages - 1));
166 BUG_ON((end_pfn) & (pageblock_nr_pages - 1));
167
168 for (pfn = start_pfn;
169 pfn < end_pfn;
170 pfn += pageblock_nr_pages) {
171 page = __first_valid_page(pfn, pageblock_nr_pages);
Wen Congyangb023f462012-12-11 16:00:45 -0800172 if (page &&
173 set_migratetype_isolate(page, skip_hwpoisoned_pages)) {
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700174 undo_pfn = pfn;
175 goto undo;
176 }
177 }
178 return 0;
179undo:
180 for (pfn = start_pfn;
KAMEZAWA Hiroyukidbc0e4c2007-11-14 16:59:12 -0800181 pfn < undo_pfn;
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700182 pfn += pageblock_nr_pages)
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +0200183 unset_migratetype_isolate(pfn_to_page(pfn), migratetype);
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700184
185 return -EBUSY;
186}
187
188/*
189 * Make isolated pages available again.
190 */
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +0200191int undo_isolate_page_range(unsigned long start_pfn, unsigned long end_pfn,
192 unsigned migratetype)
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700193{
194 unsigned long pfn;
195 struct page *page;
196 BUG_ON((start_pfn) & (pageblock_nr_pages - 1));
197 BUG_ON((end_pfn) & (pageblock_nr_pages - 1));
198 for (pfn = start_pfn;
199 pfn < end_pfn;
200 pfn += pageblock_nr_pages) {
201 page = __first_valid_page(pfn, pageblock_nr_pages);
KAMEZAWA Hiroyukidbc0e4c2007-11-14 16:59:12 -0800202 if (!page || get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700203 continue;
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +0200204 unset_migratetype_isolate(page, migratetype);
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700205 }
206 return 0;
207}
208/*
209 * Test all pages in the range is free(means isolated) or not.
210 * all pages in [start_pfn...end_pfn) must be in the same zone.
211 * zone->lock must be held before call this.
212 *
Michal Nazarewicz0815f3d2012-04-03 15:06:15 +0200213 * Returns 1 if all pages in the range are isolated.
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700214 */
215static int
Wen Congyangb023f462012-12-11 16:00:45 -0800216__test_page_isolated_in_pageblock(unsigned long pfn, unsigned long end_pfn,
217 bool skip_hwpoisoned_pages)
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700218{
219 struct page *page;
220
221 while (pfn < end_pfn) {
222 if (!pfn_valid_within(pfn)) {
223 pfn++;
224 continue;
225 }
226 page = pfn_to_page(pfn);
Vlastimil Babkaaa016d12015-09-08 15:01:22 -0700227 if (PageBuddy(page))
Minchan Kim435b4052012-10-08 16:32:16 -0700228 /*
Vlastimil Babkaaa016d12015-09-08 15:01:22 -0700229 * If the page is on a free list, it has to be on
230 * the correct MIGRATE_ISOLATE freelist. There is no
231 * simple way to verify that as VM_BUG_ON(), though.
Minchan Kim435b4052012-10-08 16:32:16 -0700232 */
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700233 pfn += 1 << page_order(page);
Vlastimil Babkaaa016d12015-09-08 15:01:22 -0700234 else if (skip_hwpoisoned_pages && PageHWPoison(page))
235 /* A HWPoisoned page cannot be also PageBuddy */
Wen Congyangb023f462012-12-11 16:00:45 -0800236 pfn++;
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700237 else
238 break;
239 }
240 if (pfn < end_pfn)
241 return 0;
242 return 1;
243}
244
Wen Congyangb023f462012-12-11 16:00:45 -0800245int test_pages_isolated(unsigned long start_pfn, unsigned long end_pfn,
246 bool skip_hwpoisoned_pages)
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700247{
Gerald Schaefer6c1b7f62008-10-02 14:50:16 -0700248 unsigned long pfn, flags;
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700249 struct page *page;
Gerald Schaefer6c1b7f62008-10-02 14:50:16 -0700250 struct zone *zone;
251 int ret;
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700252
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700253 /*
Tang Chen85dbe702013-06-20 18:10:19 +0800254 * Note: pageblock_nr_pages != MAX_ORDER. Then, chunks of free pages
255 * are not aligned to pageblock_nr_pages.
256 * Then we just check migratetype first.
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700257 */
258 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
259 page = __first_valid_page(pfn, pageblock_nr_pages);
KAMEZAWA Hiroyukidbc0e4c2007-11-14 16:59:12 -0800260 if (page && get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700261 break;
262 }
Gerald Schaefera70dcb92008-11-06 12:53:36 -0800263 page = __first_valid_page(start_pfn, end_pfn - start_pfn);
264 if ((pfn < end_pfn) || !page)
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700265 return -EBUSY;
Tang Chen85dbe702013-06-20 18:10:19 +0800266 /* Check all pages are free or marked as ISOLATED */
Gerald Schaefera70dcb92008-11-06 12:53:36 -0800267 zone = page_zone(page);
Gerald Schaefer6c1b7f62008-10-02 14:50:16 -0700268 spin_lock_irqsave(&zone->lock, flags);
Wen Congyangb023f462012-12-11 16:00:45 -0800269 ret = __test_page_isolated_in_pageblock(start_pfn, end_pfn,
270 skip_hwpoisoned_pages);
Gerald Schaefer6c1b7f62008-10-02 14:50:16 -0700271 spin_unlock_irqrestore(&zone->lock, flags);
272 return ret ? 0 : -EBUSY;
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -0700273}
Minchan Kim723a0642012-10-08 16:32:52 -0700274
275struct page *alloc_migrate_target(struct page *page, unsigned long private,
276 int **resultp)
277{
278 gfp_t gfp_mask = GFP_USER | __GFP_MOVABLE;
279
Naoya Horiguchic8721bb2013-09-11 14:22:09 -0700280 /*
281 * TODO: allocate a destination hugepage from a nearest neighbor node,
282 * accordance with memory policy of the user process if possible. For
283 * now as a simple work-around, we use the next node for destination.
284 */
285 if (PageHuge(page)) {
286 nodemask_t src = nodemask_of_node(page_to_nid(page));
287 nodemask_t dst;
288 nodes_complement(dst, src);
289 return alloc_huge_page_node(page_hstate(compound_head(page)),
290 next_node(page_to_nid(page), dst));
291 }
292
Minchan Kim723a0642012-10-08 16:32:52 -0700293 if (PageHighMem(page))
294 gfp_mask |= __GFP_HIGHMEM;
295
296 return alloc_page(gfp_mask);
297}