blob: e08b1659ff19c6131d3a9d82c9bab16d06bac551 [file] [log] [blame]
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001/*
2 * Copyright (C) 2009 Red Hat, Inc.
3 *
4 * This work is licensed under the terms of the GNU GPL, version 2. See
5 * the COPYING file in the top-level directory.
6 */
7
Andrew Mortonae3a8c12014-06-04 16:06:58 -07008#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -080010#include <linux/mm.h>
11#include <linux/sched.h>
12#include <linux/highmem.h>
13#include <linux/hugetlb.h>
14#include <linux/mmu_notifier.h>
15#include <linux/rmap.h>
16#include <linux/swap.h>
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -080017#include <linux/shrinker.h>
Andrea Arcangeliba761492011-01-13 15:46:58 -080018#include <linux/mm_inline.h>
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -080019#include <linux/swapops.h>
Matthew Wilcox4897c762015-09-08 14:58:45 -070020#include <linux/dax.h>
Andrea Arcangeliba761492011-01-13 15:46:58 -080021#include <linux/kthread.h>
22#include <linux/khugepaged.h>
Andrea Arcangeli878aee72011-01-13 15:47:10 -080023#include <linux/freezer.h>
Dan Williamsf25748e32016-01-15 16:56:43 -080024#include <linux/pfn_t.h>
Andrea Arcangelia664b2d2011-01-13 15:47:17 -080025#include <linux/mman.h>
Dan Williams3565fce2016-01-15 16:56:55 -080026#include <linux/memremap.h>
Ralf Baechle325adeb2012-10-15 13:44:56 +020027#include <linux/pagemap.h>
Kirill A. Shutemov49071d42016-01-15 16:54:40 -080028#include <linux/debugfs.h>
Mel Gorman4daae3b2012-11-02 11:33:45 +000029#include <linux/migrate.h>
Sasha Levin43b5fbb2013-02-22 16:32:27 -080030#include <linux/hashtable.h>
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -070031#include <linux/userfaultfd_k.h>
Vladimir Davydov33c3fc72015-09-09 15:35:45 -070032#include <linux/page_idle.h>
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -080033
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -080034#include <asm/tlb.h>
35#include <asm/pgalloc.h>
36#include "internal.h"
37
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -080038enum scan_result {
39 SCAN_FAIL,
40 SCAN_SUCCEED,
41 SCAN_PMD_NULL,
42 SCAN_EXCEED_NONE_PTE,
43 SCAN_PTE_NON_PRESENT,
44 SCAN_PAGE_RO,
45 SCAN_NO_REFERENCED_PAGE,
46 SCAN_PAGE_NULL,
47 SCAN_SCAN_ABORT,
48 SCAN_PAGE_COUNT,
49 SCAN_PAGE_LRU,
50 SCAN_PAGE_LOCK,
51 SCAN_PAGE_ANON,
Kirill A. Shutemovb1caa952016-01-15 16:52:39 -080052 SCAN_PAGE_COMPOUND,
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -080053 SCAN_ANY_PROCESS,
54 SCAN_VMA_NULL,
55 SCAN_VMA_CHECK,
56 SCAN_ADDRESS_RANGE,
57 SCAN_SWAP_CACHE_PAGE,
58 SCAN_DEL_PAGE_LRU,
59 SCAN_ALLOC_HUGE_PAGE_FAIL,
60 SCAN_CGROUP_CHARGE_FAIL
61};
62
63#define CREATE_TRACE_POINTS
64#include <trace/events/huge_memory.h>
65
Andrea Arcangeliba761492011-01-13 15:46:58 -080066/*
Jianguo Wu8bfa3f92013-11-12 15:07:16 -080067 * By default transparent hugepage support is disabled in order that avoid
68 * to risk increase the memory footprint of applications without a guaranteed
69 * benefit. When transparent hugepage support is enabled, is for all mappings,
70 * and khugepaged scans all mappings.
71 * Defrag is invoked by khugepaged hugepage allocations and by page faults
72 * for all hugepage allocations.
Andrea Arcangeliba761492011-01-13 15:46:58 -080073 */
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -080074unsigned long transparent_hugepage_flags __read_mostly =
Andrea Arcangeli13ece882011-01-13 15:47:07 -080075#ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
Andrea Arcangeliba761492011-01-13 15:46:58 -080076 (1<<TRANSPARENT_HUGEPAGE_FLAG)|
Andrea Arcangeli13ece882011-01-13 15:47:07 -080077#endif
78#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
79 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
80#endif
Mel Gorman444eb2a42016-03-17 14:19:23 -070081 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
Kirill A. Shutemov79da5402012-12-12 13:51:12 -080082 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
83 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
Andrea Arcangeliba761492011-01-13 15:46:58 -080084
85/* default scan 8*512 pte (or vmas) every 30 second */
86static unsigned int khugepaged_pages_to_scan __read_mostly = HPAGE_PMD_NR*8;
87static unsigned int khugepaged_pages_collapsed;
88static unsigned int khugepaged_full_scans;
89static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;
90/* during fragmentation poll the hugepage allocator once every minute */
91static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;
92static struct task_struct *khugepaged_thread __read_mostly;
93static DEFINE_MUTEX(khugepaged_mutex);
94static DEFINE_SPINLOCK(khugepaged_mm_lock);
95static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
96/*
97 * default collapse hugepages if there is at least one pte mapped like
98 * it would have happened if the vma was large enough during page
99 * fault.
100 */
101static unsigned int khugepaged_max_ptes_none __read_mostly = HPAGE_PMD_NR-1;
102
103static int khugepaged(void *none);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800104static int khugepaged_slab_init(void);
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700105static void khugepaged_slab_exit(void);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800106
Sasha Levin43b5fbb2013-02-22 16:32:27 -0800107#define MM_SLOTS_HASH_BITS 10
108static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
109
Andrea Arcangeliba761492011-01-13 15:46:58 -0800110static struct kmem_cache *mm_slot_cache __read_mostly;
111
112/**
113 * struct mm_slot - hash lookup from mm to mm_slot
114 * @hash: hash collision list
115 * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head
116 * @mm: the mm that this information is valid for
117 */
118struct mm_slot {
119 struct hlist_node hash;
120 struct list_head mm_node;
121 struct mm_struct *mm;
122};
123
124/**
125 * struct khugepaged_scan - cursor for scanning
126 * @mm_head: the head of the mm list to scan
127 * @mm_slot: the current mm_slot we are scanning
128 * @address: the next address inside that to be scanned
129 *
130 * There is only the one khugepaged_scan instance of this cursor structure.
131 */
132struct khugepaged_scan {
133 struct list_head mm_head;
134 struct mm_slot *mm_slot;
135 unsigned long address;
H Hartley Sweeten2f1da642011-10-31 17:09:25 -0700136};
137static struct khugepaged_scan khugepaged_scan = {
Andrea Arcangeliba761492011-01-13 15:46:58 -0800138 .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),
139};
140
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800141static struct shrinker deferred_split_shrinker;
Andrea Arcangelif0005652011-01-13 15:47:04 -0800142
Nicholas Krause2c0b80d2015-09-08 15:00:33 -0700143static void set_recommended_min_free_kbytes(void)
Andrea Arcangelif0005652011-01-13 15:47:04 -0800144{
145 struct zone *zone;
146 int nr_zones = 0;
147 unsigned long recommended_min;
Andrea Arcangelif0005652011-01-13 15:47:04 -0800148
Andrea Arcangelif0005652011-01-13 15:47:04 -0800149 for_each_populated_zone(zone)
150 nr_zones++;
151
Mel Gorman974a7862015-11-06 16:28:34 -0800152 /* Ensure 2 pageblocks are free to assist fragmentation avoidance */
Andrea Arcangelif0005652011-01-13 15:47:04 -0800153 recommended_min = pageblock_nr_pages * nr_zones * 2;
154
155 /*
156 * Make sure that on average at least two pageblocks are almost free
157 * of another type, one for a migratetype to fall back to and a
158 * second to avoid subsequent fallbacks of other types There are 3
159 * MIGRATE_TYPES we care about.
160 */
161 recommended_min += pageblock_nr_pages * nr_zones *
162 MIGRATE_PCPTYPES * MIGRATE_PCPTYPES;
163
164 /* don't ever allow to reserve more than 5% of the lowmem */
165 recommended_min = min(recommended_min,
166 (unsigned long) nr_free_buffer_pages() / 20);
167 recommended_min <<= (PAGE_SHIFT-10);
168
Han Pingtian42aa83c2014-01-23 15:53:28 -0800169 if (recommended_min > min_free_kbytes) {
170 if (user_min_free_kbytes >= 0)
171 pr_info("raising min_free_kbytes from %d to %lu "
172 "to help transparent hugepage allocations\n",
173 min_free_kbytes, recommended_min);
174
Andrea Arcangelif0005652011-01-13 15:47:04 -0800175 min_free_kbytes = recommended_min;
Han Pingtian42aa83c2014-01-23 15:53:28 -0800176 }
Andrea Arcangelif0005652011-01-13 15:47:04 -0800177 setup_per_zone_wmarks();
Andrea Arcangelif0005652011-01-13 15:47:04 -0800178}
Andrea Arcangelif0005652011-01-13 15:47:04 -0800179
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700180static int start_stop_khugepaged(void)
Andrea Arcangeliba761492011-01-13 15:46:58 -0800181{
182 int err = 0;
183 if (khugepaged_enabled()) {
Andrea Arcangeliba761492011-01-13 15:46:58 -0800184 if (!khugepaged_thread)
185 khugepaged_thread = kthread_run(khugepaged, NULL,
186 "khugepaged");
Viresh Kumar18e8e5c2015-08-12 15:59:46 +0530187 if (IS_ERR(khugepaged_thread)) {
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700188 pr_err("khugepaged: kthread_run(khugepaged) failed\n");
Andrea Arcangeliba761492011-01-13 15:46:58 -0800189 err = PTR_ERR(khugepaged_thread);
190 khugepaged_thread = NULL;
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700191 goto fail;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800192 }
Xiao Guangrong911891a2012-10-08 16:29:41 -0700193
194 if (!list_empty(&khugepaged_scan.mm_head))
Andrea Arcangeliba761492011-01-13 15:46:58 -0800195 wake_up_interruptible(&khugepaged_wait);
Andrea Arcangelif0005652011-01-13 15:47:04 -0800196
197 set_recommended_min_free_kbytes();
Xiao Guangrong911891a2012-10-08 16:29:41 -0700198 } else if (khugepaged_thread) {
Xiao Guangrong911891a2012-10-08 16:29:41 -0700199 kthread_stop(khugepaged_thread);
200 khugepaged_thread = NULL;
201 }
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700202fail:
Andrea Arcangeliba761492011-01-13 15:46:58 -0800203 return err;
204}
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800205
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800206static atomic_t huge_zero_refcount;
Wang, Yalin56873f42015-02-11 15:24:51 -0800207struct page *huge_zero_page __read_mostly;
Kirill A. Shutemov4a6c1292012-12-12 13:50:47 -0800208
Matthew Wilcoxfc437042015-09-08 14:58:51 -0700209struct page *get_huge_zero_page(void)
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800210{
211 struct page *zero_page;
212retry:
213 if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
Jason Low4db0c3c2015-04-15 16:14:08 -0700214 return READ_ONCE(huge_zero_page);
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800215
216 zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
217 HPAGE_PMD_ORDER);
Kirill A. Shutemovd8a8e1f2012-12-12 13:51:09 -0800218 if (!zero_page) {
219 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700220 return NULL;
Kirill A. Shutemovd8a8e1f2012-12-12 13:51:09 -0800221 }
222 count_vm_event(THP_ZERO_PAGE_ALLOC);
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800223 preempt_disable();
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700224 if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800225 preempt_enable();
Yu Zhao5ddacbe2014-10-29 14:50:26 -0700226 __free_pages(zero_page, compound_order(zero_page));
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800227 goto retry;
228 }
229
230 /* We take additional reference here. It will be put back by shrinker */
231 atomic_set(&huge_zero_refcount, 2);
232 preempt_enable();
Jason Low4db0c3c2015-04-15 16:14:08 -0700233 return READ_ONCE(huge_zero_page);
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800234}
235
236static void put_huge_zero_page(void)
237{
238 /*
239 * Counter should never go to zero here. Only shrinker can put
240 * last reference.
241 */
242 BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
243}
244
Glauber Costa48896462013-08-28 10:18:15 +1000245static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
246 struct shrink_control *sc)
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800247{
Glauber Costa48896462013-08-28 10:18:15 +1000248 /* we can free zero page only if last reference remains */
249 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
250}
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800251
Glauber Costa48896462013-08-28 10:18:15 +1000252static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
253 struct shrink_control *sc)
254{
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800255 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700256 struct page *zero_page = xchg(&huge_zero_page, NULL);
257 BUG_ON(zero_page == NULL);
Yu Zhao5ddacbe2014-10-29 14:50:26 -0700258 __free_pages(zero_page, compound_order(zero_page));
Glauber Costa48896462013-08-28 10:18:15 +1000259 return HPAGE_PMD_NR;
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800260 }
261
262 return 0;
263}
264
265static struct shrinker huge_zero_page_shrinker = {
Glauber Costa48896462013-08-28 10:18:15 +1000266 .count_objects = shrink_huge_zero_page_count,
267 .scan_objects = shrink_huge_zero_page_scan,
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800268 .seeks = DEFAULT_SEEKS,
269};
270
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800271#ifdef CONFIG_SYSFS
Andrea Arcangeliba761492011-01-13 15:46:58 -0800272
Mel Gorman444eb2a42016-03-17 14:19:23 -0700273static ssize_t triple_flag_store(struct kobject *kobj,
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800274 struct kobj_attribute *attr,
275 const char *buf, size_t count,
276 enum transparent_hugepage_flag enabled,
Mel Gorman444eb2a42016-03-17 14:19:23 -0700277 enum transparent_hugepage_flag deferred,
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800278 enum transparent_hugepage_flag req_madv)
279{
Mel Gorman444eb2a42016-03-17 14:19:23 -0700280 if (!memcmp("defer", buf,
281 min(sizeof("defer")-1, count))) {
282 if (enabled == deferred)
283 return -EINVAL;
284 clear_bit(enabled, &transparent_hugepage_flags);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800285 clear_bit(req_madv, &transparent_hugepage_flags);
Mel Gorman444eb2a42016-03-17 14:19:23 -0700286 set_bit(deferred, &transparent_hugepage_flags);
287 } else if (!memcmp("always", buf,
288 min(sizeof("always")-1, count))) {
289 clear_bit(deferred, &transparent_hugepage_flags);
290 clear_bit(req_madv, &transparent_hugepage_flags);
291 set_bit(enabled, &transparent_hugepage_flags);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800292 } else if (!memcmp("madvise", buf,
293 min(sizeof("madvise")-1, count))) {
294 clear_bit(enabled, &transparent_hugepage_flags);
Mel Gorman444eb2a42016-03-17 14:19:23 -0700295 clear_bit(deferred, &transparent_hugepage_flags);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800296 set_bit(req_madv, &transparent_hugepage_flags);
297 } else if (!memcmp("never", buf,
298 min(sizeof("never")-1, count))) {
299 clear_bit(enabled, &transparent_hugepage_flags);
300 clear_bit(req_madv, &transparent_hugepage_flags);
Mel Gorman444eb2a42016-03-17 14:19:23 -0700301 clear_bit(deferred, &transparent_hugepage_flags);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800302 } else
303 return -EINVAL;
304
305 return count;
306}
307
308static ssize_t enabled_show(struct kobject *kobj,
309 struct kobj_attribute *attr, char *buf)
310{
Mel Gorman444eb2a42016-03-17 14:19:23 -0700311 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
312 return sprintf(buf, "[always] madvise never\n");
313 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags))
314 return sprintf(buf, "always [madvise] never\n");
315 else
316 return sprintf(buf, "always madvise [never]\n");
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800317}
Mel Gorman444eb2a42016-03-17 14:19:23 -0700318
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800319static ssize_t enabled_store(struct kobject *kobj,
320 struct kobj_attribute *attr,
321 const char *buf, size_t count)
322{
Andrea Arcangeliba761492011-01-13 15:46:58 -0800323 ssize_t ret;
324
Mel Gorman444eb2a42016-03-17 14:19:23 -0700325 ret = triple_flag_store(kobj, attr, buf, count,
326 TRANSPARENT_HUGEPAGE_FLAG,
Andrea Arcangeliba761492011-01-13 15:46:58 -0800327 TRANSPARENT_HUGEPAGE_FLAG,
328 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
329
330 if (ret > 0) {
Xiao Guangrong911891a2012-10-08 16:29:41 -0700331 int err;
332
333 mutex_lock(&khugepaged_mutex);
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700334 err = start_stop_khugepaged();
Xiao Guangrong911891a2012-10-08 16:29:41 -0700335 mutex_unlock(&khugepaged_mutex);
336
Andrea Arcangeliba761492011-01-13 15:46:58 -0800337 if (err)
338 ret = err;
339 }
340
341 return ret;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800342}
343static struct kobj_attribute enabled_attr =
344 __ATTR(enabled, 0644, enabled_show, enabled_store);
345
346static ssize_t single_flag_show(struct kobject *kobj,
347 struct kobj_attribute *attr, char *buf,
348 enum transparent_hugepage_flag flag)
349{
Ben Hutchingse27e6152011-04-14 15:22:21 -0700350 return sprintf(buf, "%d\n",
351 !!test_bit(flag, &transparent_hugepage_flags));
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800352}
Ben Hutchingse27e6152011-04-14 15:22:21 -0700353
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800354static ssize_t single_flag_store(struct kobject *kobj,
355 struct kobj_attribute *attr,
356 const char *buf, size_t count,
357 enum transparent_hugepage_flag flag)
358{
Ben Hutchingse27e6152011-04-14 15:22:21 -0700359 unsigned long value;
360 int ret;
361
362 ret = kstrtoul(buf, 10, &value);
363 if (ret < 0)
364 return ret;
365 if (value > 1)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800366 return -EINVAL;
367
Ben Hutchingse27e6152011-04-14 15:22:21 -0700368 if (value)
369 set_bit(flag, &transparent_hugepage_flags);
370 else
371 clear_bit(flag, &transparent_hugepage_flags);
372
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800373 return count;
374}
375
376/*
377 * Currently defrag only disables __GFP_NOWAIT for allocation. A blind
378 * __GFP_REPEAT is too aggressive, it's never worth swapping tons of
379 * memory just to allocate one more hugepage.
380 */
381static ssize_t defrag_show(struct kobject *kobj,
382 struct kobj_attribute *attr, char *buf)
383{
Mel Gorman444eb2a42016-03-17 14:19:23 -0700384 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
385 return sprintf(buf, "[always] defer madvise never\n");
386 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
387 return sprintf(buf, "always [defer] madvise never\n");
388 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
389 return sprintf(buf, "always defer [madvise] never\n");
390 else
391 return sprintf(buf, "always defer madvise [never]\n");
392
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800393}
394static ssize_t defrag_store(struct kobject *kobj,
395 struct kobj_attribute *attr,
396 const char *buf, size_t count)
397{
Mel Gorman444eb2a42016-03-17 14:19:23 -0700398 return triple_flag_store(kobj, attr, buf, count,
399 TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
400 TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800401 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
402}
403static struct kobj_attribute defrag_attr =
404 __ATTR(defrag, 0644, defrag_show, defrag_store);
405
Kirill A. Shutemov79da5402012-12-12 13:51:12 -0800406static ssize_t use_zero_page_show(struct kobject *kobj,
407 struct kobj_attribute *attr, char *buf)
408{
409 return single_flag_show(kobj, attr, buf,
410 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
411}
412static ssize_t use_zero_page_store(struct kobject *kobj,
413 struct kobj_attribute *attr, const char *buf, size_t count)
414{
415 return single_flag_store(kobj, attr, buf, count,
416 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
417}
418static struct kobj_attribute use_zero_page_attr =
419 __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800420#ifdef CONFIG_DEBUG_VM
421static ssize_t debug_cow_show(struct kobject *kobj,
422 struct kobj_attribute *attr, char *buf)
423{
424 return single_flag_show(kobj, attr, buf,
425 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
426}
427static ssize_t debug_cow_store(struct kobject *kobj,
428 struct kobj_attribute *attr,
429 const char *buf, size_t count)
430{
431 return single_flag_store(kobj, attr, buf, count,
432 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
433}
434static struct kobj_attribute debug_cow_attr =
435 __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
436#endif /* CONFIG_DEBUG_VM */
437
438static struct attribute *hugepage_attr[] = {
439 &enabled_attr.attr,
440 &defrag_attr.attr,
Kirill A. Shutemov79da5402012-12-12 13:51:12 -0800441 &use_zero_page_attr.attr,
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800442#ifdef CONFIG_DEBUG_VM
443 &debug_cow_attr.attr,
444#endif
445 NULL,
446};
447
448static struct attribute_group hugepage_attr_group = {
449 .attrs = hugepage_attr,
Andrea Arcangeliba761492011-01-13 15:46:58 -0800450};
451
452static ssize_t scan_sleep_millisecs_show(struct kobject *kobj,
453 struct kobj_attribute *attr,
454 char *buf)
455{
456 return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs);
457}
458
459static ssize_t scan_sleep_millisecs_store(struct kobject *kobj,
460 struct kobj_attribute *attr,
461 const char *buf, size_t count)
462{
463 unsigned long msecs;
464 int err;
465
Jingoo Han3dbb95f2013-09-11 14:20:25 -0700466 err = kstrtoul(buf, 10, &msecs);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800467 if (err || msecs > UINT_MAX)
468 return -EINVAL;
469
470 khugepaged_scan_sleep_millisecs = msecs;
471 wake_up_interruptible(&khugepaged_wait);
472
473 return count;
474}
475static struct kobj_attribute scan_sleep_millisecs_attr =
476 __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show,
477 scan_sleep_millisecs_store);
478
479static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj,
480 struct kobj_attribute *attr,
481 char *buf)
482{
483 return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs);
484}
485
486static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj,
487 struct kobj_attribute *attr,
488 const char *buf, size_t count)
489{
490 unsigned long msecs;
491 int err;
492
Jingoo Han3dbb95f2013-09-11 14:20:25 -0700493 err = kstrtoul(buf, 10, &msecs);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800494 if (err || msecs > UINT_MAX)
495 return -EINVAL;
496
497 khugepaged_alloc_sleep_millisecs = msecs;
498 wake_up_interruptible(&khugepaged_wait);
499
500 return count;
501}
502static struct kobj_attribute alloc_sleep_millisecs_attr =
503 __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show,
504 alloc_sleep_millisecs_store);
505
506static ssize_t pages_to_scan_show(struct kobject *kobj,
507 struct kobj_attribute *attr,
508 char *buf)
509{
510 return sprintf(buf, "%u\n", khugepaged_pages_to_scan);
511}
512static ssize_t pages_to_scan_store(struct kobject *kobj,
513 struct kobj_attribute *attr,
514 const char *buf, size_t count)
515{
516 int err;
517 unsigned long pages;
518
Jingoo Han3dbb95f2013-09-11 14:20:25 -0700519 err = kstrtoul(buf, 10, &pages);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800520 if (err || !pages || pages > UINT_MAX)
521 return -EINVAL;
522
523 khugepaged_pages_to_scan = pages;
524
525 return count;
526}
527static struct kobj_attribute pages_to_scan_attr =
528 __ATTR(pages_to_scan, 0644, pages_to_scan_show,
529 pages_to_scan_store);
530
531static ssize_t pages_collapsed_show(struct kobject *kobj,
532 struct kobj_attribute *attr,
533 char *buf)
534{
535 return sprintf(buf, "%u\n", khugepaged_pages_collapsed);
536}
537static struct kobj_attribute pages_collapsed_attr =
538 __ATTR_RO(pages_collapsed);
539
540static ssize_t full_scans_show(struct kobject *kobj,
541 struct kobj_attribute *attr,
542 char *buf)
543{
544 return sprintf(buf, "%u\n", khugepaged_full_scans);
545}
546static struct kobj_attribute full_scans_attr =
547 __ATTR_RO(full_scans);
548
549static ssize_t khugepaged_defrag_show(struct kobject *kobj,
550 struct kobj_attribute *attr, char *buf)
551{
552 return single_flag_show(kobj, attr, buf,
553 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
554}
555static ssize_t khugepaged_defrag_store(struct kobject *kobj,
556 struct kobj_attribute *attr,
557 const char *buf, size_t count)
558{
559 return single_flag_store(kobj, attr, buf, count,
560 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
561}
562static struct kobj_attribute khugepaged_defrag_attr =
563 __ATTR(defrag, 0644, khugepaged_defrag_show,
564 khugepaged_defrag_store);
565
566/*
567 * max_ptes_none controls if khugepaged should collapse hugepages over
568 * any unmapped ptes in turn potentially increasing the memory
569 * footprint of the vmas. When max_ptes_none is 0 khugepaged will not
570 * reduce the available free memory in the system as it
571 * runs. Increasing max_ptes_none will instead potentially reduce the
572 * free memory in the system during the khugepaged scan.
573 */
574static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj,
575 struct kobj_attribute *attr,
576 char *buf)
577{
578 return sprintf(buf, "%u\n", khugepaged_max_ptes_none);
579}
580static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj,
581 struct kobj_attribute *attr,
582 const char *buf, size_t count)
583{
584 int err;
585 unsigned long max_ptes_none;
586
Jingoo Han3dbb95f2013-09-11 14:20:25 -0700587 err = kstrtoul(buf, 10, &max_ptes_none);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800588 if (err || max_ptes_none > HPAGE_PMD_NR-1)
589 return -EINVAL;
590
591 khugepaged_max_ptes_none = max_ptes_none;
592
593 return count;
594}
595static struct kobj_attribute khugepaged_max_ptes_none_attr =
596 __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show,
597 khugepaged_max_ptes_none_store);
598
599static struct attribute *khugepaged_attr[] = {
600 &khugepaged_defrag_attr.attr,
601 &khugepaged_max_ptes_none_attr.attr,
602 &pages_to_scan_attr.attr,
603 &pages_collapsed_attr.attr,
604 &full_scans_attr.attr,
605 &scan_sleep_millisecs_attr.attr,
606 &alloc_sleep_millisecs_attr.attr,
607 NULL,
608};
609
610static struct attribute_group khugepaged_attr_group = {
611 .attrs = khugepaged_attr,
612 .name = "khugepaged",
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800613};
Shaohua Li569e5592012-01-12 17:19:11 -0800614
615static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
616{
617 int err;
618
619 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
620 if (unlikely(!*hugepage_kobj)) {
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700621 pr_err("failed to create transparent hugepage kobject\n");
Shaohua Li569e5592012-01-12 17:19:11 -0800622 return -ENOMEM;
623 }
624
625 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
626 if (err) {
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700627 pr_err("failed to register transparent hugepage group\n");
Shaohua Li569e5592012-01-12 17:19:11 -0800628 goto delete_obj;
629 }
630
631 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
632 if (err) {
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700633 pr_err("failed to register transparent hugepage group\n");
Shaohua Li569e5592012-01-12 17:19:11 -0800634 goto remove_hp_group;
635 }
636
637 return 0;
638
639remove_hp_group:
640 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
641delete_obj:
642 kobject_put(*hugepage_kobj);
643 return err;
644}
645
646static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
647{
648 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
649 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
650 kobject_put(hugepage_kobj);
651}
652#else
653static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
654{
655 return 0;
656}
657
658static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
659{
660}
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800661#endif /* CONFIG_SYSFS */
662
663static int __init hugepage_init(void)
664{
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800665 int err;
Shaohua Li569e5592012-01-12 17:19:11 -0800666 struct kobject *hugepage_kobj;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800667
Andrea Arcangeli4b7167b2011-01-13 15:47:09 -0800668 if (!has_transparent_hugepage()) {
669 transparent_hugepage_flags = 0;
Shaohua Li569e5592012-01-12 17:19:11 -0800670 return -EINVAL;
Andrea Arcangeli4b7167b2011-01-13 15:47:09 -0800671 }
672
Shaohua Li569e5592012-01-12 17:19:11 -0800673 err = hugepage_init_sysfs(&hugepage_kobj);
674 if (err)
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700675 goto err_sysfs;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800676
677 err = khugepaged_slab_init();
678 if (err)
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700679 goto err_slab;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800680
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700681 err = register_shrinker(&huge_zero_page_shrinker);
682 if (err)
683 goto err_hzp_shrinker;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800684 err = register_shrinker(&deferred_split_shrinker);
685 if (err)
686 goto err_split_shrinker;
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800687
Rik van Riel97562cd2011-01-13 15:47:12 -0800688 /*
689 * By default disable transparent hugepages on smaller systems,
690 * where the extra memory used could hurt more than TLB overhead
691 * is likely to save. The admin can still enable it through /sys.
692 */
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700693 if (totalram_pages < (512 << (20 - PAGE_SHIFT))) {
Rik van Riel97562cd2011-01-13 15:47:12 -0800694 transparent_hugepage_flags = 0;
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700695 return 0;
696 }
Rik van Riel97562cd2011-01-13 15:47:12 -0800697
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700698 err = start_stop_khugepaged();
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700699 if (err)
700 goto err_khugepaged;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800701
Shaohua Li569e5592012-01-12 17:19:11 -0800702 return 0;
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700703err_khugepaged:
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800704 unregister_shrinker(&deferred_split_shrinker);
705err_split_shrinker:
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700706 unregister_shrinker(&huge_zero_page_shrinker);
707err_hzp_shrinker:
708 khugepaged_slab_exit();
709err_slab:
Shaohua Li569e5592012-01-12 17:19:11 -0800710 hugepage_exit_sysfs(hugepage_kobj);
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700711err_sysfs:
Andrea Arcangeliba761492011-01-13 15:46:58 -0800712 return err;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800713}
Paul Gortmakera64fb3c2014-01-23 15:53:30 -0800714subsys_initcall(hugepage_init);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800715
716static int __init setup_transparent_hugepage(char *str)
717{
718 int ret = 0;
719 if (!str)
720 goto out;
721 if (!strcmp(str, "always")) {
722 set_bit(TRANSPARENT_HUGEPAGE_FLAG,
723 &transparent_hugepage_flags);
724 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
725 &transparent_hugepage_flags);
726 ret = 1;
727 } else if (!strcmp(str, "madvise")) {
728 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
729 &transparent_hugepage_flags);
730 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
731 &transparent_hugepage_flags);
732 ret = 1;
733 } else if (!strcmp(str, "never")) {
734 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
735 &transparent_hugepage_flags);
736 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
737 &transparent_hugepage_flags);
738 ret = 1;
739 }
740out:
741 if (!ret)
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700742 pr_warn("transparent_hugepage= cannot parse, ignored\n");
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800743 return ret;
744}
745__setup("transparent_hugepage=", setup_transparent_hugepage);
746
Mel Gormanb32967f2012-11-19 12:35:47 +0000747pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800748{
749 if (likely(vma->vm_flags & VM_WRITE))
750 pmd = pmd_mkwrite(pmd);
751 return pmd;
752}
753
Kirill A. Shutemov31223592013-09-12 15:14:01 -0700754static inline pmd_t mk_huge_pmd(struct page *page, pgprot_t prot)
Bob Liub3092b32012-12-11 16:00:41 -0800755{
756 pmd_t entry;
Kirill A. Shutemov31223592013-09-12 15:14:01 -0700757 entry = mk_pmd(page, prot);
Bob Liub3092b32012-12-11 16:00:41 -0800758 entry = pmd_mkhuge(entry);
759 return entry;
760}
761
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800762static inline struct list_head *page_deferred_list(struct page *page)
763{
764 /*
765 * ->lru in the tail pages is occupied by compound_head.
766 * Let's use ->mapping + ->index in the second tail page as list_head.
767 */
768 return (struct list_head *)&page[2].mapping;
769}
770
771void prep_transhuge_page(struct page *page)
772{
773 /*
774 * we use page->mapping and page->indexlru in second tail page
775 * as list_head: assuming THP order >= 2
776 */
777 BUILD_BUG_ON(HPAGE_PMD_ORDER < 2);
778
779 INIT_LIST_HEAD(page_deferred_list(page));
780 set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
781}
782
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800783static int __do_huge_pmd_anonymous_page(struct mm_struct *mm,
784 struct vm_area_struct *vma,
Andrea Arcangeli230c92a2015-09-04 15:47:20 -0700785 unsigned long address, pmd_t *pmd,
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700786 struct page *page, gfp_t gfp,
787 unsigned int flags)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800788{
Johannes Weiner00501b52014-08-08 14:19:20 -0700789 struct mem_cgroup *memcg;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800790 pgtable_t pgtable;
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -0800791 spinlock_t *ptl;
Andrea Arcangeli230c92a2015-09-04 15:47:20 -0700792 unsigned long haddr = address & HPAGE_PMD_MASK;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800793
Sasha Levin309381fea2014-01-23 15:52:54 -0800794 VM_BUG_ON_PAGE(!PageCompound(page), page);
Johannes Weiner00501b52014-08-08 14:19:20 -0700795
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800796 if (mem_cgroup_try_charge(page, mm, gfp, &memcg, true)) {
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700797 put_page(page);
798 count_vm_event(THP_FAULT_FALLBACK);
799 return VM_FAULT_FALLBACK;
800 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800801
Johannes Weiner00501b52014-08-08 14:19:20 -0700802 pgtable = pte_alloc_one(mm, haddr);
803 if (unlikely(!pgtable)) {
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800804 mem_cgroup_cancel_charge(page, memcg, true);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700805 put_page(page);
Johannes Weiner00501b52014-08-08 14:19:20 -0700806 return VM_FAULT_OOM;
807 }
808
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800809 clear_huge_page(page, haddr, HPAGE_PMD_NR);
Minchan Kim52f37622013-04-29 15:08:15 -0700810 /*
811 * The memory barrier inside __SetPageUptodate makes sure that
812 * clear_huge_page writes become visible before the set_pmd_at()
813 * write.
814 */
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800815 __SetPageUptodate(page);
816
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -0800817 ptl = pmd_lock(mm, pmd);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800818 if (unlikely(!pmd_none(*pmd))) {
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -0800819 spin_unlock(ptl);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800820 mem_cgroup_cancel_charge(page, memcg, true);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800821 put_page(page);
822 pte_free(mm, pgtable);
823 } else {
824 pmd_t entry;
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700825
826 /* Deliver the page fault to userland */
827 if (userfaultfd_missing(vma)) {
828 int ret;
829
830 spin_unlock(ptl);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800831 mem_cgroup_cancel_charge(page, memcg, true);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700832 put_page(page);
833 pte_free(mm, pgtable);
Andrea Arcangeli230c92a2015-09-04 15:47:20 -0700834 ret = handle_userfault(vma, address, flags,
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700835 VM_UFFD_MISSING);
836 VM_BUG_ON(ret & VM_FAULT_FALLBACK);
837 return ret;
838 }
839
Kirill A. Shutemov31223592013-09-12 15:14:01 -0700840 entry = mk_huge_pmd(page, vma->vm_page_prot);
841 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -0800842 page_add_new_anon_rmap(page, vma, haddr, true);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800843 mem_cgroup_commit_charge(page, memcg, false, true);
Johannes Weiner00501b52014-08-08 14:19:20 -0700844 lru_cache_add_active_or_unevictable(page, vma);
Aneesh Kumar K.V6b0b50b2013-06-05 17:14:02 -0700845 pgtable_trans_huge_deposit(mm, pmd, pgtable);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800846 set_pmd_at(mm, haddr, pmd, entry);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800847 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
Kirill A. Shutemove1f56c82013-11-14 14:30:48 -0800848 atomic_long_inc(&mm->nr_ptes);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -0800849 spin_unlock(ptl);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700850 count_vm_event(THP_FAULT_ALLOC);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800851 }
852
David Rientjesaa2e8782012-05-29 15:06:17 -0700853 return 0;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800854}
855
Mel Gorman444eb2a42016-03-17 14:19:23 -0700856/*
857 * If THP is set to always then directly reclaim/compact as necessary
858 * If set to defer then do no reclaim and defer to khugepaged
859 * If set to madvise and the VMA is flagged then directly reclaim/compact
860 */
861static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma)
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -0800862{
Mel Gorman444eb2a42016-03-17 14:19:23 -0700863 gfp_t reclaim_flags = 0;
864
865 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags) &&
866 (vma->vm_flags & VM_HUGEPAGE))
867 reclaim_flags = __GFP_DIRECT_RECLAIM;
868 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
869 reclaim_flags = __GFP_KSWAPD_RECLAIM;
870 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
871 reclaim_flags = __GFP_DIRECT_RECLAIM;
872
873 return GFP_TRANSHUGE | reclaim_flags;
874}
875
876/* Defrag for khugepaged will enter direct reclaim/compaction if necessary */
877static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void)
878{
879 return GFP_TRANSHUGE | (khugepaged_defrag() ? __GFP_DIRECT_RECLAIM : 0);
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -0800880}
881
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -0800882/* Caller must hold page table lock. */
Kirill A. Shutemovd295e342015-09-08 14:59:34 -0700883static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800884 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700885 struct page *zero_page)
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -0800886{
887 pmd_t entry;
Andrew Morton7c414162015-09-08 14:58:43 -0700888 if (!pmd_none(*pmd))
889 return false;
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700890 entry = mk_pmd(zero_page, vma->vm_page_prot);
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -0800891 entry = pmd_mkhuge(entry);
Matthew Wilcox12c9d702016-02-02 16:57:57 -0800892 if (pgtable)
893 pgtable_trans_huge_deposit(mm, pmd, pgtable);
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -0800894 set_pmd_at(mm, haddr, pmd, entry);
Kirill A. Shutemove1f56c82013-11-14 14:30:48 -0800895 atomic_long_inc(&mm->nr_ptes);
Andrew Morton7c414162015-09-08 14:58:43 -0700896 return true;
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -0800897}
898
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800899int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma,
900 unsigned long address, pmd_t *pmd,
901 unsigned int flags)
902{
Aneesh Kumar K.V077fcf12015-02-11 15:27:12 -0800903 gfp_t gfp;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800904 struct page *page;
905 unsigned long haddr = address & HPAGE_PMD_MASK;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800906
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700907 if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
Kirill A. Shutemovc0292552013-09-12 15:14:05 -0700908 return VM_FAULT_FALLBACK;
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700909 if (unlikely(anon_vma_prepare(vma)))
910 return VM_FAULT_OOM;
David Rientjes6d50e602014-10-29 14:50:31 -0700911 if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700912 return VM_FAULT_OOM;
Dominik Dingel593befa2014-10-23 12:07:44 +0200913 if (!(flags & FAULT_FLAG_WRITE) && !mm_forbids_zeropage(mm) &&
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700914 transparent_hugepage_use_zero_page()) {
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -0800915 spinlock_t *ptl;
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700916 pgtable_t pgtable;
917 struct page *zero_page;
918 bool set;
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700919 int ret;
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700920 pgtable = pte_alloc_one(mm, haddr);
921 if (unlikely(!pgtable))
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800922 return VM_FAULT_OOM;
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700923 zero_page = get_huge_zero_page();
924 if (unlikely(!zero_page)) {
925 pte_free(mm, pgtable);
Andi Kleen81ab4202011-04-14 15:22:06 -0700926 count_vm_event(THP_FAULT_FALLBACK);
Kirill A. Shutemovc0292552013-09-12 15:14:05 -0700927 return VM_FAULT_FALLBACK;
Andi Kleen81ab4202011-04-14 15:22:06 -0700928 }
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -0800929 ptl = pmd_lock(mm, pmd);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700930 ret = 0;
931 set = false;
932 if (pmd_none(*pmd)) {
933 if (userfaultfd_missing(vma)) {
934 spin_unlock(ptl);
Andrea Arcangeli230c92a2015-09-04 15:47:20 -0700935 ret = handle_userfault(vma, address, flags,
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700936 VM_UFFD_MISSING);
937 VM_BUG_ON(ret & VM_FAULT_FALLBACK);
938 } else {
939 set_huge_zero_page(pgtable, mm, vma,
940 haddr, pmd,
941 zero_page);
942 spin_unlock(ptl);
943 set = true;
944 }
945 } else
946 spin_unlock(ptl);
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700947 if (!set) {
948 pte_free(mm, pgtable);
949 put_huge_zero_page();
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -0800950 }
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700951 return ret;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800952 }
Mel Gorman444eb2a42016-03-17 14:19:23 -0700953 gfp = alloc_hugepage_direct_gfpmask(vma);
Aneesh Kumar K.V077fcf12015-02-11 15:27:12 -0800954 page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700955 if (unlikely(!page)) {
956 count_vm_event(THP_FAULT_FALLBACK);
Kirill A. Shutemovc0292552013-09-12 15:14:05 -0700957 return VM_FAULT_FALLBACK;
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700958 }
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800959 prep_transhuge_page(page);
Andrea Arcangeli230c92a2015-09-04 15:47:20 -0700960 return __do_huge_pmd_anonymous_page(mm, vma, address, pmd, page, gfp,
961 flags);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800962}
963
Matthew Wilcoxae18d6d2015-09-08 14:59:14 -0700964static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
Dan Williamsf25748e32016-01-15 16:56:43 -0800965 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write)
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700966{
967 struct mm_struct *mm = vma->vm_mm;
968 pmd_t entry;
969 spinlock_t *ptl;
970
971 ptl = pmd_lock(mm, pmd);
Dan Williamsf25748e32016-01-15 16:56:43 -0800972 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot));
973 if (pfn_t_devmap(pfn))
974 entry = pmd_mkdevmap(entry);
Ross Zwisler01871e52016-01-15 16:56:02 -0800975 if (write) {
976 entry = pmd_mkyoung(pmd_mkdirty(entry));
977 entry = maybe_pmd_mkwrite(entry, vma);
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700978 }
Ross Zwisler01871e52016-01-15 16:56:02 -0800979 set_pmd_at(mm, addr, pmd, entry);
980 update_mmu_cache_pmd(vma, addr, pmd);
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700981 spin_unlock(ptl);
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700982}
983
984int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
Dan Williamsf25748e32016-01-15 16:56:43 -0800985 pmd_t *pmd, pfn_t pfn, bool write)
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700986{
987 pgprot_t pgprot = vma->vm_page_prot;
988 /*
989 * If we had pmd_special, we could avoid all these restrictions,
990 * but we need to be consistent with PTEs and architectures that
991 * can't support a 'special' bit.
992 */
993 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
994 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
995 (VM_PFNMAP|VM_MIXEDMAP));
996 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
Dan Williamsf25748e32016-01-15 16:56:43 -0800997 BUG_ON(!pfn_t_devmap(pfn));
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700998
999 if (addr < vma->vm_start || addr >= vma->vm_end)
1000 return VM_FAULT_SIGBUS;
1001 if (track_pfn_insert(vma, &pgprot, pfn))
1002 return VM_FAULT_SIGBUS;
Matthew Wilcoxae18d6d2015-09-08 14:59:14 -07001003 insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write);
1004 return VM_FAULT_NOPAGE;
Matthew Wilcox5cad4652015-09-08 14:58:54 -07001005}
1006
Dan Williams3565fce2016-01-15 16:56:55 -08001007static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
1008 pmd_t *pmd)
1009{
1010 pmd_t _pmd;
1011
1012 /*
1013 * We should set the dirty bit only for FOLL_WRITE but for now
1014 * the dirty bit in the pmd is meaningless. And if the dirty
1015 * bit will become meaningful and we'll only set it with
1016 * FOLL_WRITE, an atomic set_bit will be required on the pmd to
1017 * set the young bit, instead of the current set_pmd_at.
1018 */
1019 _pmd = pmd_mkyoung(pmd_mkdirty(*pmd));
1020 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
1021 pmd, _pmd, 1))
1022 update_mmu_cache_pmd(vma, addr, pmd);
1023}
1024
1025struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
1026 pmd_t *pmd, int flags)
1027{
1028 unsigned long pfn = pmd_pfn(*pmd);
1029 struct mm_struct *mm = vma->vm_mm;
1030 struct dev_pagemap *pgmap;
1031 struct page *page;
1032
1033 assert_spin_locked(pmd_lockptr(mm, pmd));
1034
1035 if (flags & FOLL_WRITE && !pmd_write(*pmd))
1036 return NULL;
1037
1038 if (pmd_present(*pmd) && pmd_devmap(*pmd))
1039 /* pass */;
1040 else
1041 return NULL;
1042
1043 if (flags & FOLL_TOUCH)
1044 touch_pmd(vma, addr, pmd);
1045
1046 /*
1047 * device mapped pages can only be returned if the
1048 * caller will manage the page reference count.
1049 */
1050 if (!(flags & FOLL_GET))
1051 return ERR_PTR(-EEXIST);
1052
1053 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;
1054 pgmap = get_dev_pagemap(pfn, NULL);
1055 if (!pgmap)
1056 return ERR_PTR(-EFAULT);
1057 page = pfn_to_page(pfn);
1058 get_page(page);
1059 put_dev_pagemap(pgmap);
1060
1061 return page;
1062}
1063
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001064int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1065 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
1066 struct vm_area_struct *vma)
1067{
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001068 spinlock_t *dst_ptl, *src_ptl;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001069 struct page *src_page;
1070 pmd_t pmd;
Matthew Wilcox12c9d702016-02-02 16:57:57 -08001071 pgtable_t pgtable = NULL;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001072 int ret;
1073
Matthew Wilcox12c9d702016-02-02 16:57:57 -08001074 if (!vma_is_dax(vma)) {
1075 ret = -ENOMEM;
1076 pgtable = pte_alloc_one(dst_mm, addr);
1077 if (unlikely(!pgtable))
1078 goto out;
1079 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001080
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001081 dst_ptl = pmd_lock(dst_mm, dst_pmd);
1082 src_ptl = pmd_lockptr(src_mm, src_pmd);
1083 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001084
1085 ret = -EAGAIN;
1086 pmd = *src_pmd;
Dan Williams5c7fb562016-01-15 16:56:52 -08001087 if (unlikely(!pmd_trans_huge(pmd) && !pmd_devmap(pmd))) {
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001088 pte_free(dst_mm, pgtable);
1089 goto out_unlock;
1090 }
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -08001091 /*
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001092 * When page table lock is held, the huge zero pmd should not be
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -08001093 * under splitting since we don't split the page itself, only pmd to
1094 * a page table.
1095 */
1096 if (is_huge_zero_pmd(pmd)) {
Kirill A. Shutemov5918d102013-04-29 15:08:44 -07001097 struct page *zero_page;
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -08001098 /*
1099 * get_huge_zero_page() will never allocate a new page here,
1100 * since we already have a zero page to copy. It just takes a
1101 * reference.
1102 */
Kirill A. Shutemov5918d102013-04-29 15:08:44 -07001103 zero_page = get_huge_zero_page();
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -07001104 set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd,
Kirill A. Shutemov5918d102013-04-29 15:08:44 -07001105 zero_page);
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -08001106 ret = 0;
1107 goto out_unlock;
1108 }
Mel Gormande466bd2013-12-18 17:08:42 -08001109
Matthew Wilcox12c9d702016-02-02 16:57:57 -08001110 if (!vma_is_dax(vma)) {
Dan Williams5c7fb562016-01-15 16:56:52 -08001111 /* thp accounting separate from pmd_devmap accounting */
1112 src_page = pmd_page(pmd);
1113 VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
1114 get_page(src_page);
1115 page_dup_rmap(src_page, true);
1116 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1117 atomic_long_inc(&dst_mm->nr_ptes);
1118 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
1119 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001120
1121 pmdp_set_wrprotect(src_mm, addr, src_pmd);
1122 pmd = pmd_mkold(pmd_wrprotect(pmd));
1123 set_pmd_at(dst_mm, addr, dst_pmd, pmd);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001124
1125 ret = 0;
1126out_unlock:
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001127 spin_unlock(src_ptl);
1128 spin_unlock(dst_ptl);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001129out:
1130 return ret;
1131}
1132
Will Deacona1dd4502012-12-11 16:01:27 -08001133void huge_pmd_set_accessed(struct mm_struct *mm,
1134 struct vm_area_struct *vma,
1135 unsigned long address,
1136 pmd_t *pmd, pmd_t orig_pmd,
1137 int dirty)
1138{
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001139 spinlock_t *ptl;
Will Deacona1dd4502012-12-11 16:01:27 -08001140 pmd_t entry;
1141 unsigned long haddr;
1142
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001143 ptl = pmd_lock(mm, pmd);
Will Deacona1dd4502012-12-11 16:01:27 -08001144 if (unlikely(!pmd_same(*pmd, orig_pmd)))
1145 goto unlock;
1146
1147 entry = pmd_mkyoung(orig_pmd);
1148 haddr = address & HPAGE_PMD_MASK;
1149 if (pmdp_set_access_flags(vma, haddr, pmd, entry, dirty))
1150 update_mmu_cache_pmd(vma, address, pmd);
1151
1152unlock:
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001153 spin_unlock(ptl);
Will Deacona1dd4502012-12-11 16:01:27 -08001154}
1155
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001156static int do_huge_pmd_wp_page_fallback(struct mm_struct *mm,
1157 struct vm_area_struct *vma,
1158 unsigned long address,
1159 pmd_t *pmd, pmd_t orig_pmd,
1160 struct page *page,
1161 unsigned long haddr)
1162{
Johannes Weiner00501b52014-08-08 14:19:20 -07001163 struct mem_cgroup *memcg;
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001164 spinlock_t *ptl;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001165 pgtable_t pgtable;
1166 pmd_t _pmd;
1167 int ret = 0, i;
1168 struct page **pages;
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001169 unsigned long mmun_start; /* For mmu_notifiers */
1170 unsigned long mmun_end; /* For mmu_notifiers */
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001171
1172 pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR,
1173 GFP_KERNEL);
1174 if (unlikely(!pages)) {
1175 ret |= VM_FAULT_OOM;
1176 goto out;
1177 }
1178
1179 for (i = 0; i < HPAGE_PMD_NR; i++) {
Andi Kleencc5d4622011-03-22 16:33:13 -07001180 pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE |
1181 __GFP_OTHER_NODE,
Andi Kleen19ee1512011-03-04 17:36:31 -08001182 vma, address, page_to_nid(page));
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001183 if (unlikely(!pages[i] ||
Johannes Weiner00501b52014-08-08 14:19:20 -07001184 mem_cgroup_try_charge(pages[i], mm, GFP_KERNEL,
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08001185 &memcg, false))) {
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001186 if (pages[i])
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001187 put_page(pages[i]);
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001188 while (--i >= 0) {
Johannes Weiner00501b52014-08-08 14:19:20 -07001189 memcg = (void *)page_private(pages[i]);
1190 set_page_private(pages[i], 0);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08001191 mem_cgroup_cancel_charge(pages[i], memcg,
1192 false);
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001193 put_page(pages[i]);
1194 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001195 kfree(pages);
1196 ret |= VM_FAULT_OOM;
1197 goto out;
1198 }
Johannes Weiner00501b52014-08-08 14:19:20 -07001199 set_page_private(pages[i], (unsigned long)memcg);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001200 }
1201
1202 for (i = 0; i < HPAGE_PMD_NR; i++) {
1203 copy_user_highpage(pages[i], page + i,
Hillf Danton0089e482011-10-31 17:09:38 -07001204 haddr + PAGE_SIZE * i, vma);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001205 __SetPageUptodate(pages[i]);
1206 cond_resched();
1207 }
1208
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001209 mmun_start = haddr;
1210 mmun_end = haddr + HPAGE_PMD_SIZE;
1211 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
1212
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001213 ptl = pmd_lock(mm, pmd);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001214 if (unlikely(!pmd_same(*pmd, orig_pmd)))
1215 goto out_free_pages;
Sasha Levin309381fea2014-01-23 15:52:54 -08001216 VM_BUG_ON_PAGE(!PageHead(page), page);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001217
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -07001218 pmdp_huge_clear_flush_notify(vma, haddr, pmd);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001219 /* leave pmd empty until pte is filled */
1220
Aneesh Kumar K.V6b0b50b2013-06-05 17:14:02 -07001221 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001222 pmd_populate(mm, &_pmd, pgtable);
1223
1224 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
1225 pte_t *pte, entry;
1226 entry = mk_pte(pages[i], vma->vm_page_prot);
1227 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
Johannes Weiner00501b52014-08-08 14:19:20 -07001228 memcg = (void *)page_private(pages[i]);
1229 set_page_private(pages[i], 0);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08001230 page_add_new_anon_rmap(pages[i], vma, haddr, false);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08001231 mem_cgroup_commit_charge(pages[i], memcg, false, false);
Johannes Weiner00501b52014-08-08 14:19:20 -07001232 lru_cache_add_active_or_unevictable(pages[i], vma);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001233 pte = pte_offset_map(&_pmd, haddr);
1234 VM_BUG_ON(!pte_none(*pte));
1235 set_pte_at(mm, haddr, pte, entry);
1236 pte_unmap(pte);
1237 }
1238 kfree(pages);
1239
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001240 smp_wmb(); /* make pte visible before pmd */
1241 pmd_populate(mm, pmd, pgtable);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08001242 page_remove_rmap(page, true);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001243 spin_unlock(ptl);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001244
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001245 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1246
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001247 ret |= VM_FAULT_WRITE;
1248 put_page(page);
1249
1250out:
1251 return ret;
1252
1253out_free_pages:
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001254 spin_unlock(ptl);
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001255 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001256 for (i = 0; i < HPAGE_PMD_NR; i++) {
Johannes Weiner00501b52014-08-08 14:19:20 -07001257 memcg = (void *)page_private(pages[i]);
1258 set_page_private(pages[i], 0);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08001259 mem_cgroup_cancel_charge(pages[i], memcg, false);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001260 put_page(pages[i]);
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001261 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001262 kfree(pages);
1263 goto out;
1264}
1265
1266int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma,
1267 unsigned long address, pmd_t *pmd, pmd_t orig_pmd)
1268{
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001269 spinlock_t *ptl;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001270 int ret = 0;
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001271 struct page *page = NULL, *new_page;
Johannes Weiner00501b52014-08-08 14:19:20 -07001272 struct mem_cgroup *memcg;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001273 unsigned long haddr;
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001274 unsigned long mmun_start; /* For mmu_notifiers */
1275 unsigned long mmun_end; /* For mmu_notifiers */
Michal Hocko3b363692015-04-15 16:13:29 -07001276 gfp_t huge_gfp; /* for allocation and charge */
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001277
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001278 ptl = pmd_lockptr(mm, pmd);
Sasha Levin81d1b092014-10-09 15:28:10 -07001279 VM_BUG_ON_VMA(!vma->anon_vma, vma);
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001280 haddr = address & HPAGE_PMD_MASK;
1281 if (is_huge_zero_pmd(orig_pmd))
1282 goto alloc;
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001283 spin_lock(ptl);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001284 if (unlikely(!pmd_same(*pmd, orig_pmd)))
1285 goto out_unlock;
1286
1287 page = pmd_page(orig_pmd);
Sasha Levin309381fea2014-01-23 15:52:54 -08001288 VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page);
Kirill A. Shutemov1f25fe22016-01-15 16:52:24 -08001289 /*
1290 * We can only reuse the page if nobody else maps the huge page or it's
1291 * part. We can do it by checking page_mapcount() on each sub-page, but
1292 * it's expensive.
1293 * The cheaper way is to check page_count() to be equal 1: every
1294 * mapcount takes page reference reference, so this way we can
1295 * guarantee, that the PMD is the only mapping.
1296 * This can give false negative if somebody pinned the page, but that's
1297 * fine.
1298 */
1299 if (page_mapcount(page) == 1 && page_count(page) == 1) {
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001300 pmd_t entry;
1301 entry = pmd_mkyoung(orig_pmd);
1302 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
1303 if (pmdp_set_access_flags(vma, haddr, pmd, entry, 1))
David Millerb113da62012-10-08 16:34:25 -07001304 update_mmu_cache_pmd(vma, address, pmd);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001305 ret |= VM_FAULT_WRITE;
1306 goto out_unlock;
1307 }
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08001308 get_page(page);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001309 spin_unlock(ptl);
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001310alloc:
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001311 if (transparent_hugepage_enabled(vma) &&
Aneesh Kumar K.V077fcf12015-02-11 15:27:12 -08001312 !transparent_hugepage_debug_cow()) {
Mel Gorman444eb2a42016-03-17 14:19:23 -07001313 huge_gfp = alloc_hugepage_direct_gfpmask(vma);
Michal Hocko3b363692015-04-15 16:13:29 -07001314 new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
Aneesh Kumar K.V077fcf12015-02-11 15:27:12 -08001315 } else
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001316 new_page = NULL;
1317
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08001318 if (likely(new_page)) {
1319 prep_transhuge_page(new_page);
1320 } else {
Hugh Dickinseecc1e42014-01-12 01:25:21 -08001321 if (!page) {
Kirill A. Shutemov78ddc532016-01-15 16:52:42 -08001322 split_huge_pmd(vma, pmd, address);
Kirill A. Shutemove9b71ca2014-04-03 14:48:17 -07001323 ret |= VM_FAULT_FALLBACK;
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001324 } else {
1325 ret = do_huge_pmd_wp_page_fallback(mm, vma, address,
1326 pmd, orig_pmd, page, haddr);
Kirill A. Shutemov9845cbb2014-02-25 15:01:42 -08001327 if (ret & VM_FAULT_OOM) {
Kirill A. Shutemov78ddc532016-01-15 16:52:42 -08001328 split_huge_pmd(vma, pmd, address);
Kirill A. Shutemov9845cbb2014-02-25 15:01:42 -08001329 ret |= VM_FAULT_FALLBACK;
1330 }
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08001331 put_page(page);
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001332 }
David Rientjes17766dd2013-09-12 15:14:06 -07001333 count_vm_event(THP_FAULT_FALLBACK);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001334 goto out;
1335 }
1336
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08001337 if (unlikely(mem_cgroup_try_charge(new_page, mm, huge_gfp, &memcg,
1338 true))) {
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001339 put_page(new_page);
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001340 if (page) {
Kirill A. Shutemov78ddc532016-01-15 16:52:42 -08001341 split_huge_pmd(vma, pmd, address);
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08001342 put_page(page);
Kirill A. Shutemov9845cbb2014-02-25 15:01:42 -08001343 } else
Kirill A. Shutemov78ddc532016-01-15 16:52:42 -08001344 split_huge_pmd(vma, pmd, address);
Kirill A. Shutemov9845cbb2014-02-25 15:01:42 -08001345 ret |= VM_FAULT_FALLBACK;
David Rientjes17766dd2013-09-12 15:14:06 -07001346 count_vm_event(THP_FAULT_FALLBACK);
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001347 goto out;
1348 }
1349
David Rientjes17766dd2013-09-12 15:14:06 -07001350 count_vm_event(THP_FAULT_ALLOC);
1351
Hugh Dickinseecc1e42014-01-12 01:25:21 -08001352 if (!page)
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001353 clear_huge_page(new_page, haddr, HPAGE_PMD_NR);
1354 else
1355 copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001356 __SetPageUptodate(new_page);
1357
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001358 mmun_start = haddr;
1359 mmun_end = haddr + HPAGE_PMD_SIZE;
1360 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
1361
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001362 spin_lock(ptl);
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001363 if (page)
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08001364 put_page(page);
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001365 if (unlikely(!pmd_same(*pmd, orig_pmd))) {
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001366 spin_unlock(ptl);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08001367 mem_cgroup_cancel_charge(new_page, memcg, true);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001368 put_page(new_page);
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001369 goto out_mn;
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001370 } else {
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001371 pmd_t entry;
Kirill A. Shutemov31223592013-09-12 15:14:01 -07001372 entry = mk_huge_pmd(new_page, vma->vm_page_prot);
1373 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -07001374 pmdp_huge_clear_flush_notify(vma, haddr, pmd);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08001375 page_add_new_anon_rmap(new_page, vma, haddr, true);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08001376 mem_cgroup_commit_charge(new_page, memcg, false, true);
Johannes Weiner00501b52014-08-08 14:19:20 -07001377 lru_cache_add_active_or_unevictable(new_page, vma);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001378 set_pmd_at(mm, haddr, pmd, entry);
David Millerb113da62012-10-08 16:34:25 -07001379 update_mmu_cache_pmd(vma, address, pmd);
Hugh Dickinseecc1e42014-01-12 01:25:21 -08001380 if (!page) {
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001381 add_mm_counter(mm, MM_ANONPAGES, HPAGE_PMD_NR);
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -08001382 put_huge_zero_page();
1383 } else {
Sasha Levin309381fea2014-01-23 15:52:54 -08001384 VM_BUG_ON_PAGE(!PageHead(page), page);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08001385 page_remove_rmap(page, true);
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001386 put_page(page);
1387 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001388 ret |= VM_FAULT_WRITE;
1389 }
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001390 spin_unlock(ptl);
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001391out_mn:
1392 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
1393out:
1394 return ret;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001395out_unlock:
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001396 spin_unlock(ptl);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001397 return ret;
1398}
1399
David Rientjesb676b292012-10-08 16:34:03 -07001400struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001401 unsigned long addr,
1402 pmd_t *pmd,
1403 unsigned int flags)
1404{
David Rientjesb676b292012-10-08 16:34:03 -07001405 struct mm_struct *mm = vma->vm_mm;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001406 struct page *page = NULL;
1407
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001408 assert_spin_locked(pmd_lockptr(mm, pmd));
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001409
1410 if (flags & FOLL_WRITE && !pmd_write(*pmd))
1411 goto out;
1412
Kirill A. Shutemov85facf22013-02-04 14:28:42 -08001413 /* Avoid dumping huge zero page */
1414 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
1415 return ERR_PTR(-EFAULT);
1416
Mel Gorman2b4847e2013-12-18 17:08:32 -08001417 /* Full NUMA hinting faults to serialise migration in fault paths */
Mel Gorman8a0516e2015-02-12 14:58:22 -08001418 if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
Mel Gorman2b4847e2013-12-18 17:08:32 -08001419 goto out;
1420
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001421 page = pmd_page(*pmd);
Sasha Levin309381fea2014-01-23 15:52:54 -08001422 VM_BUG_ON_PAGE(!PageHead(page), page);
Dan Williams3565fce2016-01-15 16:56:55 -08001423 if (flags & FOLL_TOUCH)
1424 touch_pmd(vma, addr, pmd);
Eric B Munsonde60f5f2015-11-05 18:51:36 -08001425 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
Kirill A. Shutemove90309c2016-01-15 16:54:33 -08001426 /*
1427 * We don't mlock() pte-mapped THPs. This way we can avoid
1428 * leaking mlocked pages into non-VM_LOCKED VMAs.
1429 *
1430 * In most cases the pmd is the only mapping of the page as we
1431 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for
1432 * writable private mappings in populate_vma_page_range().
1433 *
1434 * The only scenario when we have the page shared here is if we
1435 * mlocking read-only mapping shared over fork(). We skip
1436 * mlocking such pages.
1437 */
1438 if (compound_mapcount(page) == 1 && !PageDoubleMap(page) &&
1439 page->mapping && trylock_page(page)) {
David Rientjesb676b292012-10-08 16:34:03 -07001440 lru_add_drain();
1441 if (page->mapping)
1442 mlock_vma_page(page);
1443 unlock_page(page);
1444 }
1445 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001446 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
Sasha Levin309381fea2014-01-23 15:52:54 -08001447 VM_BUG_ON_PAGE(!PageCompound(page), page);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001448 if (flags & FOLL_GET)
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08001449 get_page(page);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001450
1451out:
1452 return page;
1453}
1454
Mel Gormand10e63f2012-10-25 14:16:31 +02001455/* NUMA hinting page fault entry point for trans huge pmds */
Mel Gorman4daae3b2012-11-02 11:33:45 +00001456int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma,
1457 unsigned long addr, pmd_t pmd, pmd_t *pmdp)
Mel Gormand10e63f2012-10-25 14:16:31 +02001458{
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001459 spinlock_t *ptl;
Mel Gormanb8916632013-10-07 11:28:44 +01001460 struct anon_vma *anon_vma = NULL;
Mel Gormanb32967f2012-11-19 12:35:47 +00001461 struct page *page;
Mel Gormand10e63f2012-10-25 14:16:31 +02001462 unsigned long haddr = addr & HPAGE_PMD_MASK;
Mel Gorman8191acb2013-10-07 11:28:45 +01001463 int page_nid = -1, this_nid = numa_node_id();
Peter Zijlstra90572892013-10-07 11:29:20 +01001464 int target_nid, last_cpupid = -1;
Mel Gorman8191acb2013-10-07 11:28:45 +01001465 bool page_locked;
1466 bool migrated = false;
Mel Gormanb191f9b2015-03-25 15:55:40 -07001467 bool was_writable;
Peter Zijlstra6688cc02013-10-07 11:29:24 +01001468 int flags = 0;
Mel Gormand10e63f2012-10-25 14:16:31 +02001469
Mel Gormanc0e7cad2015-02-12 14:58:41 -08001470 /* A PROT_NONE fault should not end up here */
1471 BUG_ON(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)));
1472
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001473 ptl = pmd_lock(mm, pmdp);
Mel Gormand10e63f2012-10-25 14:16:31 +02001474 if (unlikely(!pmd_same(pmd, *pmdp)))
1475 goto out_unlock;
1476
Mel Gormande466bd2013-12-18 17:08:42 -08001477 /*
1478 * If there are potential migrations, wait for completion and retry
1479 * without disrupting NUMA hinting information. Do not relock and
1480 * check_same as the page may no longer be mapped.
1481 */
1482 if (unlikely(pmd_trans_migrating(*pmdp))) {
Mel Gorman5d833062015-02-12 14:58:16 -08001483 page = pmd_page(*pmdp);
Mel Gormande466bd2013-12-18 17:08:42 -08001484 spin_unlock(ptl);
Mel Gorman5d833062015-02-12 14:58:16 -08001485 wait_on_page_locked(page);
Mel Gormande466bd2013-12-18 17:08:42 -08001486 goto out;
1487 }
1488
Mel Gormand10e63f2012-10-25 14:16:31 +02001489 page = pmd_page(pmd);
Mel Gormana1a46182013-10-07 11:28:50 +01001490 BUG_ON(is_huge_zero_page(page));
Mel Gorman8191acb2013-10-07 11:28:45 +01001491 page_nid = page_to_nid(page);
Peter Zijlstra90572892013-10-07 11:29:20 +01001492 last_cpupid = page_cpupid_last(page);
Mel Gorman03c5a6e2012-11-02 14:52:48 +00001493 count_vm_numa_event(NUMA_HINT_FAULTS);
Rik van Riel04bb2f92013-10-07 11:29:36 +01001494 if (page_nid == this_nid) {
Mel Gorman03c5a6e2012-11-02 14:52:48 +00001495 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
Rik van Riel04bb2f92013-10-07 11:29:36 +01001496 flags |= TNF_FAULT_LOCAL;
1497 }
Mel Gorman4daae3b2012-11-02 11:33:45 +00001498
Mel Gormanbea66fb2015-03-25 15:55:37 -07001499 /* See similar comment in do_numa_page for explanation */
1500 if (!(vma->vm_flags & VM_WRITE))
Peter Zijlstra6688cc02013-10-07 11:29:24 +01001501 flags |= TNF_NO_GROUP;
1502
1503 /*
Mel Gormanff9042b2013-10-07 11:28:43 +01001504 * Acquire the page lock to serialise THP migrations but avoid dropping
1505 * page_table_lock if at all possible
1506 */
Mel Gormanb8916632013-10-07 11:28:44 +01001507 page_locked = trylock_page(page);
1508 target_nid = mpol_misplaced(page, vma, haddr);
1509 if (target_nid == -1) {
1510 /* If the page was locked, there are no parallel migrations */
Mel Gormana54a4072013-10-07 11:28:46 +01001511 if (page_locked)
Mel Gormanb8916632013-10-07 11:28:44 +01001512 goto clear_pmdnuma;
Mel Gorman2b4847e2013-12-18 17:08:32 -08001513 }
Mel Gorman4daae3b2012-11-02 11:33:45 +00001514
Mel Gormande466bd2013-12-18 17:08:42 -08001515 /* Migration could have started since the pmd_trans_migrating check */
Mel Gorman2b4847e2013-12-18 17:08:32 -08001516 if (!page_locked) {
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001517 spin_unlock(ptl);
Mel Gormanb8916632013-10-07 11:28:44 +01001518 wait_on_page_locked(page);
Mel Gormana54a4072013-10-07 11:28:46 +01001519 page_nid = -1;
Mel Gormanb8916632013-10-07 11:28:44 +01001520 goto out;
1521 }
1522
Mel Gorman2b4847e2013-12-18 17:08:32 -08001523 /*
1524 * Page is misplaced. Page lock serialises migrations. Acquire anon_vma
1525 * to serialises splits
1526 */
Mel Gormanb8916632013-10-07 11:28:44 +01001527 get_page(page);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001528 spin_unlock(ptl);
Mel Gormanb8916632013-10-07 11:28:44 +01001529 anon_vma = page_lock_anon_vma_read(page);
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001530
Peter Zijlstrac69307d2013-10-07 11:28:41 +01001531 /* Confirm the PMD did not change while page_table_lock was released */
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001532 spin_lock(ptl);
Mel Gormanb32967f2012-11-19 12:35:47 +00001533 if (unlikely(!pmd_same(pmd, *pmdp))) {
1534 unlock_page(page);
1535 put_page(page);
Mel Gormana54a4072013-10-07 11:28:46 +01001536 page_nid = -1;
Mel Gormanb32967f2012-11-19 12:35:47 +00001537 goto out_unlock;
1538 }
Mel Gormanff9042b2013-10-07 11:28:43 +01001539
Mel Gormanc3a489c2013-12-18 17:08:38 -08001540 /* Bail if we fail to protect against THP splits for any reason */
1541 if (unlikely(!anon_vma)) {
1542 put_page(page);
1543 page_nid = -1;
1544 goto clear_pmdnuma;
1545 }
1546
Mel Gormana54a4072013-10-07 11:28:46 +01001547 /*
1548 * Migrate the THP to the requested node, returns with page unlocked
Mel Gorman8a0516e2015-02-12 14:58:22 -08001549 * and access rights restored.
Mel Gormana54a4072013-10-07 11:28:46 +01001550 */
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001551 spin_unlock(ptl);
Mel Gormanb32967f2012-11-19 12:35:47 +00001552 migrated = migrate_misplaced_transhuge_page(mm, vma,
Hugh Dickins340ef392013-02-22 16:34:33 -08001553 pmdp, pmd, addr, page, target_nid);
Peter Zijlstra6688cc02013-10-07 11:29:24 +01001554 if (migrated) {
1555 flags |= TNF_MIGRATED;
Mel Gorman8191acb2013-10-07 11:28:45 +01001556 page_nid = target_nid;
Mel Gorman074c2382015-03-25 15:55:42 -07001557 } else
1558 flags |= TNF_MIGRATE_FAIL;
Mel Gormanb32967f2012-11-19 12:35:47 +00001559
Mel Gorman8191acb2013-10-07 11:28:45 +01001560 goto out;
Mel Gorman4daae3b2012-11-02 11:33:45 +00001561clear_pmdnuma:
Mel Gormana54a4072013-10-07 11:28:46 +01001562 BUG_ON(!PageLocked(page));
Mel Gormanb191f9b2015-03-25 15:55:40 -07001563 was_writable = pmd_write(pmd);
Mel Gorman4d942462015-02-12 14:58:28 -08001564 pmd = pmd_modify(pmd, vma->vm_page_prot);
Mel Gormanb7b04002015-03-25 15:55:45 -07001565 pmd = pmd_mkyoung(pmd);
Mel Gormanb191f9b2015-03-25 15:55:40 -07001566 if (was_writable)
1567 pmd = pmd_mkwrite(pmd);
Mel Gormand10e63f2012-10-25 14:16:31 +02001568 set_pmd_at(mm, haddr, pmdp, pmd);
Mel Gormand10e63f2012-10-25 14:16:31 +02001569 update_mmu_cache_pmd(vma, addr, pmdp);
Mel Gormana54a4072013-10-07 11:28:46 +01001570 unlock_page(page);
Mel Gormand10e63f2012-10-25 14:16:31 +02001571out_unlock:
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001572 spin_unlock(ptl);
Mel Gormanb8916632013-10-07 11:28:44 +01001573
1574out:
1575 if (anon_vma)
1576 page_unlock_anon_vma_read(anon_vma);
1577
Mel Gorman8191acb2013-10-07 11:28:45 +01001578 if (page_nid != -1)
Peter Zijlstra6688cc02013-10-07 11:29:24 +01001579 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, flags);
Mel Gorman8191acb2013-10-07 11:28:45 +01001580
Mel Gormand10e63f2012-10-25 14:16:31 +02001581 return 0;
1582}
1583
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001584int madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1585 pmd_t *pmd, unsigned long addr, unsigned long next)
1586
1587{
1588 spinlock_t *ptl;
1589 pmd_t orig_pmd;
1590 struct page *page;
1591 struct mm_struct *mm = tlb->mm;
1592 int ret = 0;
1593
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001594 ptl = pmd_trans_huge_lock(pmd, vma);
1595 if (!ptl)
Linus Torvalds25eedab2016-01-17 18:33:15 -08001596 goto out_unlocked;
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001597
1598 orig_pmd = *pmd;
1599 if (is_huge_zero_pmd(orig_pmd)) {
1600 ret = 1;
1601 goto out;
1602 }
1603
1604 page = pmd_page(orig_pmd);
1605 /*
1606 * If other processes are mapping this page, we couldn't discard
1607 * the page unless they all do MADV_FREE so let's skip the page.
1608 */
1609 if (page_mapcount(page) != 1)
1610 goto out;
1611
1612 if (!trylock_page(page))
1613 goto out;
1614
1615 /*
1616 * If user want to discard part-pages of THP, split it so MADV_FREE
1617 * will deactivate only them.
1618 */
1619 if (next - addr != HPAGE_PMD_SIZE) {
1620 get_page(page);
1621 spin_unlock(ptl);
1622 if (split_huge_page(page)) {
1623 put_page(page);
1624 unlock_page(page);
1625 goto out_unlocked;
1626 }
1627 put_page(page);
1628 unlock_page(page);
1629 ret = 1;
1630 goto out_unlocked;
1631 }
1632
1633 if (PageDirty(page))
1634 ClearPageDirty(page);
1635 unlock_page(page);
1636
1637 if (PageActive(page))
1638 deactivate_page(page);
1639
1640 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
1641 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
1642 tlb->fullmm);
1643 orig_pmd = pmd_mkold(orig_pmd);
1644 orig_pmd = pmd_mkclean(orig_pmd);
1645
1646 set_pmd_at(mm, addr, pmd, orig_pmd);
1647 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1648 }
1649 ret = 1;
1650out:
1651 spin_unlock(ptl);
1652out_unlocked:
1653 return ret;
1654}
1655
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001656int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
Shaohua Lif21760b2012-01-12 17:19:16 -08001657 pmd_t *pmd, unsigned long addr)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001658{
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001659 pmd_t orig_pmd;
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001660 spinlock_t *ptl;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001661
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001662 ptl = __pmd_trans_huge_lock(pmd, vma);
1663 if (!ptl)
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001664 return 0;
1665 /*
1666 * For architectures like ppc64 we look at deposited pgtable
1667 * when calling pmdp_huge_get_and_clear. So do the
1668 * pgtable_trans_huge_withdraw after finishing pmdp related
1669 * operations.
1670 */
1671 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
1672 tlb->fullmm);
1673 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1674 if (vma_is_dax(vma)) {
1675 spin_unlock(ptl);
1676 if (is_huge_zero_pmd(orig_pmd))
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -08001677 put_huge_zero_page();
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001678 } else if (is_huge_zero_pmd(orig_pmd)) {
1679 pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
1680 atomic_long_dec(&tlb->mm->nr_ptes);
1681 spin_unlock(ptl);
1682 put_huge_zero_page();
1683 } else {
1684 struct page *page = pmd_page(orig_pmd);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08001685 page_remove_rmap(page, true);
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001686 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
1687 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
1688 VM_BUG_ON_PAGE(!PageHead(page), page);
1689 pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
1690 atomic_long_dec(&tlb->mm->nr_ptes);
1691 spin_unlock(ptl);
1692 tlb_remove_page(tlb, page);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001693 }
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001694 return 1;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001695}
1696
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001697bool move_huge_pmd(struct vm_area_struct *vma, struct vm_area_struct *new_vma,
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001698 unsigned long old_addr,
1699 unsigned long new_addr, unsigned long old_end,
1700 pmd_t *old_pmd, pmd_t *new_pmd)
1701{
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001702 spinlock_t *old_ptl, *new_ptl;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001703 pmd_t pmd;
1704
1705 struct mm_struct *mm = vma->vm_mm;
1706
1707 if ((old_addr & ~HPAGE_PMD_MASK) ||
1708 (new_addr & ~HPAGE_PMD_MASK) ||
1709 old_end - old_addr < HPAGE_PMD_SIZE ||
1710 (new_vma->vm_flags & VM_NOHUGEPAGE))
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001711 return false;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001712
1713 /*
1714 * The destination pmd shouldn't be established, free_pgtables()
1715 * should have release it.
1716 */
1717 if (WARN_ON(!pmd_none(*new_pmd))) {
1718 VM_BUG_ON(pmd_trans_huge(*new_pmd));
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001719 return false;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001720 }
1721
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001722 /*
1723 * We don't have to worry about the ordering of src and dst
1724 * ptlocks because exclusive mmap_sem prevents deadlock.
1725 */
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001726 old_ptl = __pmd_trans_huge_lock(old_pmd, vma);
1727 if (old_ptl) {
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001728 new_ptl = pmd_lockptr(mm, new_pmd);
1729 if (new_ptl != old_ptl)
1730 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -07001731 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001732 VM_BUG_ON(!pmd_none(*new_pmd));
Kirill A. Shutemov35928062013-12-12 17:12:33 -08001733
Kirill A. Shutemov69a8ec22016-02-17 13:11:12 -08001734 if (pmd_move_must_withdraw(new_ptl, old_ptl) &&
1735 vma_is_anonymous(vma)) {
Aneesh Kumar K.Vb3084f42014-01-13 11:34:24 +05301736 pgtable_t pgtable;
Kirill A. Shutemov35928062013-12-12 17:12:33 -08001737 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
1738 pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
Kirill A. Shutemov35928062013-12-12 17:12:33 -08001739 }
Aneesh Kumar K.Vb3084f42014-01-13 11:34:24 +05301740 set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd));
1741 if (new_ptl != old_ptl)
1742 spin_unlock(new_ptl);
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001743 spin_unlock(old_ptl);
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001744 return true;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001745 }
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001746 return false;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001747}
1748
Mel Gormanf123d742013-10-07 11:28:49 +01001749/*
1750 * Returns
1751 * - 0 if PMD could not be locked
1752 * - 1 if PMD was locked but protections unchange and TLB flush unnecessary
1753 * - HPAGE_PMD_NR is protections changed and TLB flush necessary
1754 */
Johannes Weinercd7548a2011-01-13 15:47:04 -08001755int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
Mel Gormane944fd62015-02-12 14:58:35 -08001756 unsigned long addr, pgprot_t newprot, int prot_numa)
Johannes Weinercd7548a2011-01-13 15:47:04 -08001757{
1758 struct mm_struct *mm = vma->vm_mm;
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001759 spinlock_t *ptl;
Johannes Weinercd7548a2011-01-13 15:47:04 -08001760 int ret = 0;
1761
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001762 ptl = __pmd_trans_huge_lock(pmd, vma);
1763 if (ptl) {
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001764 pmd_t entry;
Mel Gormanb191f9b2015-03-25 15:55:40 -07001765 bool preserve_write = prot_numa && pmd_write(*pmd);
Mel Gormanba68bc02015-03-07 15:20:48 +00001766 ret = 1;
Mel Gormane944fd62015-02-12 14:58:35 -08001767
1768 /*
1769 * Avoid trapping faults against the zero page. The read-only
1770 * data is likely to be read-cached on the local CPU and
1771 * local/remote hits to the zero page are not interesting.
1772 */
1773 if (prot_numa && is_huge_zero_pmd(*pmd)) {
1774 spin_unlock(ptl);
Mel Gormanba68bc02015-03-07 15:20:48 +00001775 return ret;
Mel Gormane944fd62015-02-12 14:58:35 -08001776 }
1777
Mel Gorman10c10452015-02-12 14:58:44 -08001778 if (!prot_numa || !pmd_protnone(*pmd)) {
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -07001779 entry = pmdp_huge_get_and_clear_notify(mm, addr, pmd);
Mel Gorman10c10452015-02-12 14:58:44 -08001780 entry = pmd_modify(entry, newprot);
Mel Gormanb191f9b2015-03-25 15:55:40 -07001781 if (preserve_write)
1782 entry = pmd_mkwrite(entry);
Mel Gorman10c10452015-02-12 14:58:44 -08001783 ret = HPAGE_PMD_NR;
1784 set_pmd_at(mm, addr, pmd, entry);
Mel Gormanb191f9b2015-03-25 15:55:40 -07001785 BUG_ON(!preserve_write && pmd_write(entry));
Mel Gorman10c10452015-02-12 14:58:44 -08001786 }
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001787 spin_unlock(ptl);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001788 }
Johannes Weinercd7548a2011-01-13 15:47:04 -08001789
1790 return ret;
1791}
1792
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001793/*
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001794 * Returns true if a given pmd maps a thp, false otherwise.
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001795 *
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001796 * Note that if it returns true, this routine returns without unlocking page
1797 * table lock. So callers must unlock it.
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001798 */
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001799spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001800{
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001801 spinlock_t *ptl;
1802 ptl = pmd_lock(vma->vm_mm, pmd);
Dan Williams5c7fb562016-01-15 16:56:52 -08001803 if (likely(pmd_trans_huge(*pmd) || pmd_devmap(*pmd)))
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001804 return ptl;
1805 spin_unlock(ptl);
1806 return NULL;
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001807}
1808
Vlastimil Babka9050d7e2014-03-03 15:38:27 -08001809#define VM_NO_THP (VM_SPECIAL | VM_HUGETLB | VM_SHARED | VM_MAYSHARE)
Andrea Arcangeli78f11a22011-04-27 15:26:45 -07001810
Andrea Arcangeli60ab3242011-01-13 15:47:18 -08001811int hugepage_madvise(struct vm_area_struct *vma,
1812 unsigned long *vm_flags, int advice)
Andrea Arcangeli0af4e982011-01-13 15:46:55 -08001813{
Andrea Arcangelia664b2d2011-01-13 15:47:17 -08001814 switch (advice) {
1815 case MADV_HUGEPAGE:
Alex Thorlton1e1836e2014-04-07 15:37:09 -07001816#ifdef CONFIG_S390
1817 /*
1818 * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390
1819 * can't handle this properly after s390_enable_sie, so we simply
1820 * ignore the madvise to prevent qemu from causing a SIGSEGV.
1821 */
1822 if (mm_has_pgste(vma->vm_mm))
1823 return 0;
1824#endif
Andrea Arcangelia664b2d2011-01-13 15:47:17 -08001825 /*
1826 * Be somewhat over-protective like KSM for now!
1827 */
Jason J. Herne1a763612015-11-20 15:57:04 -08001828 if (*vm_flags & VM_NO_THP)
Andrea Arcangelia664b2d2011-01-13 15:47:17 -08001829 return -EINVAL;
1830 *vm_flags &= ~VM_NOHUGEPAGE;
1831 *vm_flags |= VM_HUGEPAGE;
Andrea Arcangeli60ab3242011-01-13 15:47:18 -08001832 /*
1833 * If the vma become good for khugepaged to scan,
1834 * register it here without waiting a page fault that
1835 * may not happen any time soon.
1836 */
David Rientjes6d50e602014-10-29 14:50:31 -07001837 if (unlikely(khugepaged_enter_vma_merge(vma, *vm_flags)))
Andrea Arcangeli60ab3242011-01-13 15:47:18 -08001838 return -ENOMEM;
Andrea Arcangelia664b2d2011-01-13 15:47:17 -08001839 break;
1840 case MADV_NOHUGEPAGE:
1841 /*
1842 * Be somewhat over-protective like KSM for now!
1843 */
Jason J. Herne1a763612015-11-20 15:57:04 -08001844 if (*vm_flags & VM_NO_THP)
Andrea Arcangelia664b2d2011-01-13 15:47:17 -08001845 return -EINVAL;
1846 *vm_flags &= ~VM_HUGEPAGE;
1847 *vm_flags |= VM_NOHUGEPAGE;
Andrea Arcangeli60ab3242011-01-13 15:47:18 -08001848 /*
1849 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
1850 * this vma even if we leave the mm registered in khugepaged if
1851 * it got registered before VM_NOHUGEPAGE was set.
1852 */
Andrea Arcangelia664b2d2011-01-13 15:47:17 -08001853 break;
1854 }
Andrea Arcangeli0af4e982011-01-13 15:46:55 -08001855
1856 return 0;
1857}
1858
Andrea Arcangeliba761492011-01-13 15:46:58 -08001859static int __init khugepaged_slab_init(void)
1860{
1861 mm_slot_cache = kmem_cache_create("khugepaged_mm_slot",
1862 sizeof(struct mm_slot),
1863 __alignof__(struct mm_slot), 0, NULL);
1864 if (!mm_slot_cache)
1865 return -ENOMEM;
1866
1867 return 0;
1868}
1869
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -07001870static void __init khugepaged_slab_exit(void)
1871{
1872 kmem_cache_destroy(mm_slot_cache);
1873}
1874
Andrea Arcangeliba761492011-01-13 15:46:58 -08001875static inline struct mm_slot *alloc_mm_slot(void)
1876{
1877 if (!mm_slot_cache) /* initialization failed */
1878 return NULL;
1879 return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
1880}
1881
1882static inline void free_mm_slot(struct mm_slot *mm_slot)
1883{
1884 kmem_cache_free(mm_slot_cache, mm_slot);
1885}
1886
Andrea Arcangeliba761492011-01-13 15:46:58 -08001887static struct mm_slot *get_mm_slot(struct mm_struct *mm)
1888{
1889 struct mm_slot *mm_slot;
Andrea Arcangeliba761492011-01-13 15:46:58 -08001890
Sasha Levinb67bfe02013-02-27 17:06:00 -08001891 hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm)
Andrea Arcangeliba761492011-01-13 15:46:58 -08001892 if (mm == mm_slot->mm)
1893 return mm_slot;
Sasha Levin43b5fbb2013-02-22 16:32:27 -08001894
Andrea Arcangeliba761492011-01-13 15:46:58 -08001895 return NULL;
1896}
1897
1898static void insert_to_mm_slots_hash(struct mm_struct *mm,
1899 struct mm_slot *mm_slot)
1900{
Andrea Arcangeliba761492011-01-13 15:46:58 -08001901 mm_slot->mm = mm;
Sasha Levin43b5fbb2013-02-22 16:32:27 -08001902 hash_add(mm_slots_hash, &mm_slot->hash, (long)mm);
Andrea Arcangeliba761492011-01-13 15:46:58 -08001903}
1904
1905static inline int khugepaged_test_exit(struct mm_struct *mm)
1906{
1907 return atomic_read(&mm->mm_users) == 0;
1908}
1909
1910int __khugepaged_enter(struct mm_struct *mm)
1911{
1912 struct mm_slot *mm_slot;
1913 int wakeup;
1914
1915 mm_slot = alloc_mm_slot();
1916 if (!mm_slot)
1917 return -ENOMEM;
1918
1919 /* __khugepaged_exit() must not run from under us */
Sasha Levin96dad672014-10-09 15:28:39 -07001920 VM_BUG_ON_MM(khugepaged_test_exit(mm), mm);
Andrea Arcangeliba761492011-01-13 15:46:58 -08001921 if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
1922 free_mm_slot(mm_slot);
1923 return 0;
1924 }
1925
1926 spin_lock(&khugepaged_mm_lock);
1927 insert_to_mm_slots_hash(mm, mm_slot);
1928 /*
1929 * Insert just behind the scanning cursor, to let the area settle
1930 * down a little.
1931 */
1932 wakeup = list_empty(&khugepaged_scan.mm_head);
1933 list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head);
1934 spin_unlock(&khugepaged_mm_lock);
1935
1936 atomic_inc(&mm->mm_count);
1937 if (wakeup)
1938 wake_up_interruptible(&khugepaged_wait);
1939
1940 return 0;
1941}
1942
David Rientjes6d50e602014-10-29 14:50:31 -07001943int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
1944 unsigned long vm_flags)
Andrea Arcangeliba761492011-01-13 15:46:58 -08001945{
1946 unsigned long hstart, hend;
1947 if (!vma->anon_vma)
1948 /*
1949 * Not yet faulted in so we will register later in the
1950 * page fault if needed.
1951 */
1952 return 0;
Andrea Arcangeli78f11a22011-04-27 15:26:45 -07001953 if (vma->vm_ops)
Andrea Arcangeliba761492011-01-13 15:46:58 -08001954 /* khugepaged not yet working on file or special mappings */
1955 return 0;
David Rientjes6d50e602014-10-29 14:50:31 -07001956 VM_BUG_ON_VMA(vm_flags & VM_NO_THP, vma);
Andrea Arcangeliba761492011-01-13 15:46:58 -08001957 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
1958 hend = vma->vm_end & HPAGE_PMD_MASK;
1959 if (hstart < hend)
David Rientjes6d50e602014-10-29 14:50:31 -07001960 return khugepaged_enter(vma, vm_flags);
Andrea Arcangeliba761492011-01-13 15:46:58 -08001961 return 0;
1962}
1963
1964void __khugepaged_exit(struct mm_struct *mm)
1965{
1966 struct mm_slot *mm_slot;
1967 int free = 0;
1968
1969 spin_lock(&khugepaged_mm_lock);
1970 mm_slot = get_mm_slot(mm);
1971 if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {
Sasha Levin43b5fbb2013-02-22 16:32:27 -08001972 hash_del(&mm_slot->hash);
Andrea Arcangeliba761492011-01-13 15:46:58 -08001973 list_del(&mm_slot->mm_node);
1974 free = 1;
1975 }
Chris Wrightd788e802011-07-25 17:12:14 -07001976 spin_unlock(&khugepaged_mm_lock);
Andrea Arcangeliba761492011-01-13 15:46:58 -08001977
1978 if (free) {
Andrea Arcangeliba761492011-01-13 15:46:58 -08001979 clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
1980 free_mm_slot(mm_slot);
1981 mmdrop(mm);
1982 } else if (mm_slot) {
Andrea Arcangeliba761492011-01-13 15:46:58 -08001983 /*
1984 * This is required to serialize against
1985 * khugepaged_test_exit() (which is guaranteed to run
1986 * under mmap sem read mode). Stop here (after we
1987 * return all pagetables will be destroyed) until
1988 * khugepaged has finished working on the pagetables
1989 * under the mmap_sem.
1990 */
1991 down_write(&mm->mmap_sem);
1992 up_write(&mm->mmap_sem);
Chris Wrightd788e802011-07-25 17:12:14 -07001993 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08001994}
1995
1996static void release_pte_page(struct page *page)
1997{
1998 /* 0 stands for page_is_file_cache(page) == false */
1999 dec_zone_page_state(page, NR_ISOLATED_ANON + 0);
2000 unlock_page(page);
2001 putback_lru_page(page);
2002}
2003
2004static void release_pte_pages(pte_t *pte, pte_t *_pte)
2005{
2006 while (--_pte >= pte) {
2007 pte_t pteval = *_pte;
Ebru Akagunduzca0984c2015-04-14 15:45:24 -07002008 if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)))
Andrea Arcangeliba761492011-01-13 15:46:58 -08002009 release_pte_page(pte_page(pteval));
2010 }
2011}
2012
Andrea Arcangeliba761492011-01-13 15:46:58 -08002013static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
2014 unsigned long address,
2015 pte_t *pte)
2016{
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002017 struct page *page = NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002018 pte_t *_pte;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002019 int none_or_zero = 0, result = 0;
Ebru Akagunduz10359212015-02-11 15:28:28 -08002020 bool referenced = false, writable = false;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002021
Andrea Arcangeliba761492011-01-13 15:46:58 -08002022 for (_pte = pte; _pte < pte+HPAGE_PMD_NR;
2023 _pte++, address += PAGE_SIZE) {
2024 pte_t pteval = *_pte;
Minchan Kim47aee4d2015-10-22 13:32:19 -07002025 if (pte_none(pteval) || (pte_present(pteval) &&
2026 is_zero_pfn(pte_pfn(pteval)))) {
Andrea Arcangelic1294d02015-09-04 15:46:27 -07002027 if (!userfaultfd_armed(vma) &&
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002028 ++none_or_zero <= khugepaged_max_ptes_none) {
Andrea Arcangeliba761492011-01-13 15:46:58 -08002029 continue;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002030 } else {
2031 result = SCAN_EXCEED_NONE_PTE;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002032 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002033 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002034 }
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002035 if (!pte_present(pteval)) {
2036 result = SCAN_PTE_NON_PRESENT;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002037 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002038 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002039 page = vm_normal_page(vma, address, pteval);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002040 if (unlikely(!page)) {
2041 result = SCAN_PAGE_NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002042 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002043 }
Bob Liu344aa352012-12-11 16:00:34 -08002044
Sasha Levin309381fea2014-01-23 15:52:54 -08002045 VM_BUG_ON_PAGE(PageCompound(page), page);
2046 VM_BUG_ON_PAGE(!PageAnon(page), page);
2047 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002048
Andrea Arcangeliba761492011-01-13 15:46:58 -08002049 /*
2050 * We can do it before isolate_lru_page because the
2051 * page can't be freed from under us. NOTE: PG_lock
2052 * is needed to serialize against split_huge_page
2053 * when invoked from the VM.
2054 */
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002055 if (!trylock_page(page)) {
2056 result = SCAN_PAGE_LOCK;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002057 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002058 }
Ebru Akagunduz10359212015-02-11 15:28:28 -08002059
2060 /*
2061 * cannot use mapcount: can't collapse if there's a gup pin.
2062 * The page must only be referenced by the scanned process
2063 * and page swap cache.
2064 */
2065 if (page_count(page) != 1 + !!PageSwapCache(page)) {
2066 unlock_page(page);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002067 result = SCAN_PAGE_COUNT;
Ebru Akagunduz10359212015-02-11 15:28:28 -08002068 goto out;
2069 }
2070 if (pte_write(pteval)) {
2071 writable = true;
2072 } else {
2073 if (PageSwapCache(page) && !reuse_swap_page(page)) {
2074 unlock_page(page);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002075 result = SCAN_SWAP_CACHE_PAGE;
Ebru Akagunduz10359212015-02-11 15:28:28 -08002076 goto out;
2077 }
2078 /*
2079 * Page is not in the swap cache. It can be collapsed
2080 * into a THP.
2081 */
2082 }
2083
Andrea Arcangeliba761492011-01-13 15:46:58 -08002084 /*
2085 * Isolate the page to avoid collapsing an hugepage
2086 * currently in use by the VM.
2087 */
2088 if (isolate_lru_page(page)) {
2089 unlock_page(page);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002090 result = SCAN_DEL_PAGE_LRU;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002091 goto out;
2092 }
2093 /* 0 stands for page_is_file_cache(page) == false */
2094 inc_zone_page_state(page, NR_ISOLATED_ANON + 0);
Sasha Levin309381fea2014-01-23 15:52:54 -08002095 VM_BUG_ON_PAGE(!PageLocked(page), page);
2096 VM_BUG_ON_PAGE(PageLRU(page), page);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002097
2098 /* If there is no mapped pte young don't collapse the page */
Vladimir Davydov33c3fc72015-09-09 15:35:45 -07002099 if (pte_young(pteval) ||
2100 page_is_young(page) || PageReferenced(page) ||
Andrea Arcangeli8ee53822011-01-13 15:47:10 -08002101 mmu_notifier_test_young(vma->vm_mm, address))
Ebru Akagunduz10359212015-02-11 15:28:28 -08002102 referenced = true;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002103 }
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002104 if (likely(writable)) {
2105 if (likely(referenced)) {
2106 result = SCAN_SUCCEED;
yalin wang16fd0fe2016-01-21 16:40:30 -08002107 trace_mm_collapse_huge_page_isolate(page, none_or_zero,
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002108 referenced, writable, result);
2109 return 1;
2110 }
2111 } else {
2112 result = SCAN_PAGE_RO;
2113 }
2114
Andrea Arcangeliba761492011-01-13 15:46:58 -08002115out:
Bob Liu344aa352012-12-11 16:00:34 -08002116 release_pte_pages(pte, _pte);
yalin wang16fd0fe2016-01-21 16:40:30 -08002117 trace_mm_collapse_huge_page_isolate(page, none_or_zero,
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002118 referenced, writable, result);
Bob Liu344aa352012-12-11 16:00:34 -08002119 return 0;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002120}
2121
2122static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
2123 struct vm_area_struct *vma,
2124 unsigned long address,
2125 spinlock_t *ptl)
2126{
2127 pte_t *_pte;
2128 for (_pte = pte; _pte < pte+HPAGE_PMD_NR; _pte++) {
2129 pte_t pteval = *_pte;
2130 struct page *src_page;
2131
Ebru Akagunduzca0984c2015-04-14 15:45:24 -07002132 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
Andrea Arcangeliba761492011-01-13 15:46:58 -08002133 clear_user_highpage(page, address);
2134 add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1);
Ebru Akagunduzca0984c2015-04-14 15:45:24 -07002135 if (is_zero_pfn(pte_pfn(pteval))) {
2136 /*
2137 * ptl mostly unnecessary.
2138 */
2139 spin_lock(ptl);
2140 /*
2141 * paravirt calls inside pte_clear here are
2142 * superfluous.
2143 */
2144 pte_clear(vma->vm_mm, address, _pte);
2145 spin_unlock(ptl);
2146 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002147 } else {
2148 src_page = pte_page(pteval);
2149 copy_user_highpage(page, src_page, address, vma);
Sasha Levin309381fea2014-01-23 15:52:54 -08002150 VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002151 release_pte_page(src_page);
2152 /*
2153 * ptl mostly unnecessary, but preempt has to
2154 * be disabled to update the per-cpu stats
2155 * inside page_remove_rmap().
2156 */
2157 spin_lock(ptl);
2158 /*
2159 * paravirt calls inside pte_clear here are
2160 * superfluous.
2161 */
2162 pte_clear(vma->vm_mm, address, _pte);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08002163 page_remove_rmap(src_page, false);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002164 spin_unlock(ptl);
2165 free_page_and_swap_cache(src_page);
2166 }
2167
2168 address += PAGE_SIZE;
2169 page++;
2170 }
2171}
2172
Xiao Guangrong26234f32012-10-08 16:29:51 -07002173static void khugepaged_alloc_sleep(void)
2174{
Petr Mladekbde43c62015-09-08 15:04:05 -07002175 DEFINE_WAIT(wait);
2176
2177 add_wait_queue(&khugepaged_wait, &wait);
2178 freezable_schedule_timeout_interruptible(
2179 msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));
2180 remove_wait_queue(&khugepaged_wait, &wait);
Xiao Guangrong26234f32012-10-08 16:29:51 -07002181}
2182
Bob Liu9f1b8682013-11-12 15:07:37 -08002183static int khugepaged_node_load[MAX_NUMNODES];
2184
David Rientjes14a4e212014-08-06 16:07:29 -07002185static bool khugepaged_scan_abort(int nid)
2186{
2187 int i;
2188
2189 /*
2190 * If zone_reclaim_mode is disabled, then no extra effort is made to
2191 * allocate memory locally.
2192 */
2193 if (!zone_reclaim_mode)
2194 return false;
2195
2196 /* If there is a count for this node already, it must be acceptable */
2197 if (khugepaged_node_load[nid])
2198 return false;
2199
2200 for (i = 0; i < MAX_NUMNODES; i++) {
2201 if (!khugepaged_node_load[i])
2202 continue;
2203 if (node_distance(nid, i) > RECLAIM_DISTANCE)
2204 return true;
2205 }
2206 return false;
2207}
2208
Xiao Guangrong26234f32012-10-08 16:29:51 -07002209#ifdef CONFIG_NUMA
Bob Liu9f1b8682013-11-12 15:07:37 -08002210static int khugepaged_find_target_node(void)
2211{
2212 static int last_khugepaged_target_node = NUMA_NO_NODE;
2213 int nid, target_node = 0, max_value = 0;
2214
2215 /* find first node with max normal pages hit */
2216 for (nid = 0; nid < MAX_NUMNODES; nid++)
2217 if (khugepaged_node_load[nid] > max_value) {
2218 max_value = khugepaged_node_load[nid];
2219 target_node = nid;
2220 }
2221
2222 /* do some balance if several nodes have the same hit record */
2223 if (target_node <= last_khugepaged_target_node)
2224 for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES;
2225 nid++)
2226 if (max_value == khugepaged_node_load[nid]) {
2227 target_node = nid;
2228 break;
2229 }
2230
2231 last_khugepaged_target_node = target_node;
2232 return target_node;
2233}
2234
Xiao Guangrong26234f32012-10-08 16:29:51 -07002235static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
2236{
2237 if (IS_ERR(*hpage)) {
2238 if (!*wait)
2239 return false;
2240
2241 *wait = false;
Xiao Guangronge3b41262012-10-08 16:32:57 -07002242 *hpage = NULL;
Xiao Guangrong26234f32012-10-08 16:29:51 -07002243 khugepaged_alloc_sleep();
2244 } else if (*hpage) {
2245 put_page(*hpage);
2246 *hpage = NULL;
2247 }
2248
2249 return true;
2250}
2251
Michal Hocko3b363692015-04-15 16:13:29 -07002252static struct page *
2253khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm,
Aaron Tomlind6669d62015-11-06 16:28:52 -08002254 unsigned long address, int node)
Xiao Guangrong26234f32012-10-08 16:29:51 -07002255{
Sasha Levin309381fea2014-01-23 15:52:54 -08002256 VM_BUG_ON_PAGE(*hpage, *hpage);
Vlastimil Babka8b164562014-10-09 15:27:00 -07002257
Xiao Guangrong26234f32012-10-08 16:29:51 -07002258 /*
Vlastimil Babka8b164562014-10-09 15:27:00 -07002259 * Before allocating the hugepage, release the mmap_sem read lock.
2260 * The allocation can take potentially a long time if it involves
2261 * sync compaction, and we do not need to hold the mmap_sem during
2262 * that. We will recheck the vma after taking it again in write mode.
Xiao Guangrong26234f32012-10-08 16:29:51 -07002263 */
2264 up_read(&mm->mmap_sem);
Vlastimil Babka8b164562014-10-09 15:27:00 -07002265
Vlastimil Babka96db8002015-09-08 15:03:50 -07002266 *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER);
Xiao Guangrong26234f32012-10-08 16:29:51 -07002267 if (unlikely(!*hpage)) {
2268 count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
2269 *hpage = ERR_PTR(-ENOMEM);
2270 return NULL;
2271 }
2272
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002273 prep_transhuge_page(*hpage);
Xiao Guangrong26234f32012-10-08 16:29:51 -07002274 count_vm_event(THP_COLLAPSE_ALLOC);
2275 return *hpage;
2276}
2277#else
Bob Liu9f1b8682013-11-12 15:07:37 -08002278static int khugepaged_find_target_node(void)
2279{
2280 return 0;
2281}
2282
Mel Gorman444eb2a42016-03-17 14:19:23 -07002283static inline struct page *alloc_khugepaged_hugepage(void)
Bob Liu10dc4152013-11-12 15:07:35 -08002284{
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002285 struct page *page;
2286
Mel Gorman444eb2a42016-03-17 14:19:23 -07002287 page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(),
2288 HPAGE_PMD_ORDER);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002289 if (page)
2290 prep_transhuge_page(page);
2291 return page;
Bob Liu10dc4152013-11-12 15:07:35 -08002292}
2293
Xiao Guangrong26234f32012-10-08 16:29:51 -07002294static struct page *khugepaged_alloc_hugepage(bool *wait)
2295{
2296 struct page *hpage;
2297
2298 do {
Mel Gorman444eb2a42016-03-17 14:19:23 -07002299 hpage = alloc_khugepaged_hugepage();
Xiao Guangrong26234f32012-10-08 16:29:51 -07002300 if (!hpage) {
2301 count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
2302 if (!*wait)
2303 return NULL;
2304
2305 *wait = false;
2306 khugepaged_alloc_sleep();
2307 } else
2308 count_vm_event(THP_COLLAPSE_ALLOC);
2309 } while (unlikely(!hpage) && likely(khugepaged_enabled()));
2310
2311 return hpage;
2312}
2313
2314static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
2315{
2316 if (!*hpage)
2317 *hpage = khugepaged_alloc_hugepage(wait);
2318
2319 if (unlikely(!*hpage))
2320 return false;
2321
2322 return true;
2323}
2324
Michal Hocko3b363692015-04-15 16:13:29 -07002325static struct page *
2326khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm,
Aaron Tomlind6669d62015-11-06 16:28:52 -08002327 unsigned long address, int node)
Xiao Guangrong26234f32012-10-08 16:29:51 -07002328{
2329 up_read(&mm->mmap_sem);
2330 VM_BUG_ON(!*hpage);
Michal Hocko3b363692015-04-15 16:13:29 -07002331
Xiao Guangrong26234f32012-10-08 16:29:51 -07002332 return *hpage;
2333}
2334#endif
2335
Bob Liufa475e52012-12-11 16:00:39 -08002336static bool hugepage_vma_check(struct vm_area_struct *vma)
2337{
2338 if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
2339 (vma->vm_flags & VM_NOHUGEPAGE))
2340 return false;
Bob Liufa475e52012-12-11 16:00:39 -08002341 if (!vma->anon_vma || vma->vm_ops)
2342 return false;
2343 if (is_vma_temporary_stack(vma))
2344 return false;
Sasha Levin81d1b092014-10-09 15:28:10 -07002345 VM_BUG_ON_VMA(vma->vm_flags & VM_NO_THP, vma);
Bob Liufa475e52012-12-11 16:00:39 -08002346 return true;
2347}
2348
Andrea Arcangeliba761492011-01-13 15:46:58 -08002349static void collapse_huge_page(struct mm_struct *mm,
Xiao Guangrong26234f32012-10-08 16:29:51 -07002350 unsigned long address,
2351 struct page **hpage,
2352 struct vm_area_struct *vma,
2353 int node)
Andrea Arcangeliba761492011-01-13 15:46:58 -08002354{
Andrea Arcangeliba761492011-01-13 15:46:58 -08002355 pmd_t *pmd, _pmd;
2356 pte_t *pte;
2357 pgtable_t pgtable;
2358 struct page *new_page;
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002359 spinlock_t *pmd_ptl, *pte_ptl;
Arnd Bergmann629d9d12016-01-18 21:50:26 +01002360 int isolated = 0, result = 0;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002361 unsigned long hstart, hend;
Johannes Weiner00501b52014-08-08 14:19:20 -07002362 struct mem_cgroup *memcg;
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07002363 unsigned long mmun_start; /* For mmu_notifiers */
2364 unsigned long mmun_end; /* For mmu_notifiers */
Michal Hocko3b363692015-04-15 16:13:29 -07002365 gfp_t gfp;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002366
2367 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
Andrea Arcangeli692e0b32011-05-24 17:12:14 -07002368
Michal Hocko3b363692015-04-15 16:13:29 -07002369 /* Only allocate from the target node */
Mel Gorman444eb2a42016-03-17 14:19:23 -07002370 gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_OTHER_NODE | __GFP_THISNODE;
Michal Hocko3b363692015-04-15 16:13:29 -07002371
Xiao Guangrong26234f32012-10-08 16:29:51 -07002372 /* release the mmap_sem read lock. */
Aaron Tomlind6669d62015-11-06 16:28:52 -08002373 new_page = khugepaged_alloc_page(hpage, gfp, mm, address, node);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002374 if (!new_page) {
2375 result = SCAN_ALLOC_HUGE_PAGE_FAIL;
2376 goto out_nolock;
2377 }
Andrea Arcangelice83d212011-01-13 15:47:06 -08002378
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08002379 if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg, true))) {
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002380 result = SCAN_CGROUP_CHARGE_FAIL;
2381 goto out_nolock;
2382 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002383
2384 /*
2385 * Prevent all access to pagetables with the exception of
2386 * gup_fast later hanlded by the ptep_clear_flush and the VM
2387 * handled by the anon_vma lock + PG_lock.
2388 */
2389 down_write(&mm->mmap_sem);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002390 if (unlikely(khugepaged_test_exit(mm))) {
2391 result = SCAN_ANY_PROCESS;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002392 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002393 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002394
2395 vma = find_vma(mm, address);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002396 if (!vma) {
2397 result = SCAN_VMA_NULL;
Libina8f531eb2013-09-11 14:20:38 -07002398 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002399 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002400 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
2401 hend = vma->vm_end & HPAGE_PMD_MASK;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002402 if (address < hstart || address + HPAGE_PMD_SIZE > hend) {
2403 result = SCAN_ADDRESS_RANGE;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002404 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002405 }
2406 if (!hugepage_vma_check(vma)) {
2407 result = SCAN_VMA_CHECK;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002408 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002409 }
Bob Liu62190492012-12-11 16:00:37 -08002410 pmd = mm_find_pmd(mm, address);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002411 if (!pmd) {
2412 result = SCAN_PMD_NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002413 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002414 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002415
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +00002416 anon_vma_lock_write(vma->anon_vma);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002417
2418 pte = pte_offset_map(pmd, address);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002419 pte_ptl = pte_lockptr(mm, pmd);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002420
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07002421 mmun_start = address;
2422 mmun_end = address + HPAGE_PMD_SIZE;
2423 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002424 pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
Andrea Arcangeliba761492011-01-13 15:46:58 -08002425 /*
2426 * After this gup_fast can't run anymore. This also removes
2427 * any huge TLB entry from the CPU so we won't allow
2428 * huge and small TLB entries for the same virtual address
2429 * to avoid the risk of CPU bugs in that area.
2430 */
Aneesh Kumar K.V15a25b22015-06-24 16:57:39 -07002431 _pmd = pmdp_collapse_flush(vma, address, pmd);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002432 spin_unlock(pmd_ptl);
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07002433 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002434
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002435 spin_lock(pte_ptl);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002436 isolated = __collapse_huge_page_isolate(vma, address, pte);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002437 spin_unlock(pte_ptl);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002438
2439 if (unlikely(!isolated)) {
Johannes Weiner453c7192011-01-20 14:44:18 -08002440 pte_unmap(pte);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002441 spin_lock(pmd_ptl);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002442 BUG_ON(!pmd_none(*pmd));
Aneesh Kumar K.V7c342512013-05-24 15:55:21 -07002443 /*
2444 * We can only use set_pmd_at when establishing
2445 * hugepmds and never for establishing regular pmds that
2446 * points to regular pagetables. Use pmd_populate for that
2447 */
2448 pmd_populate(mm, pmd, pmd_pgtable(_pmd));
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002449 spin_unlock(pmd_ptl);
Konstantin Khlebnikov08b52702013-02-22 16:34:40 -08002450 anon_vma_unlock_write(vma->anon_vma);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002451 result = SCAN_FAIL;
Andrea Arcangelice83d212011-01-13 15:47:06 -08002452 goto out;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002453 }
2454
2455 /*
2456 * All pages are isolated and locked so anon_vma rmap
2457 * can't run anymore.
2458 */
Konstantin Khlebnikov08b52702013-02-22 16:34:40 -08002459 anon_vma_unlock_write(vma->anon_vma);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002460
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002461 __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl);
Johannes Weiner453c7192011-01-20 14:44:18 -08002462 pte_unmap(pte);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002463 __SetPageUptodate(new_page);
2464 pgtable = pmd_pgtable(_pmd);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002465
Kirill A. Shutemov31223592013-09-12 15:14:01 -07002466 _pmd = mk_huge_pmd(new_page, vma->vm_page_prot);
2467 _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002468
2469 /*
2470 * spin_lock() below is not the equivalent of smp_wmb(), so
2471 * this is needed to avoid the copy_huge_page writes to become
2472 * visible after the set_pmd_at() write.
2473 */
2474 smp_wmb();
2475
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002476 spin_lock(pmd_ptl);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002477 BUG_ON(!pmd_none(*pmd));
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08002478 page_add_new_anon_rmap(new_page, vma, address, true);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08002479 mem_cgroup_commit_charge(new_page, memcg, false, true);
Johannes Weiner00501b52014-08-08 14:19:20 -07002480 lru_cache_add_active_or_unevictable(new_page, vma);
Aneesh Kumar K.Vfce144b2013-06-05 17:14:06 -07002481 pgtable_trans_huge_deposit(mm, pmd, pgtable);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002482 set_pmd_at(mm, address, pmd, _pmd);
David Millerb113da62012-10-08 16:34:25 -07002483 update_mmu_cache_pmd(vma, address, pmd);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002484 spin_unlock(pmd_ptl);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002485
2486 *hpage = NULL;
Xiao Guangrong420256ef2012-10-08 16:29:49 -07002487
Andrea Arcangeliba761492011-01-13 15:46:58 -08002488 khugepaged_pages_collapsed++;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002489 result = SCAN_SUCCEED;
Andrea Arcangelice83d212011-01-13 15:47:06 -08002490out_up_write:
Andrea Arcangeliba761492011-01-13 15:46:58 -08002491 up_write(&mm->mmap_sem);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002492 trace_mm_collapse_huge_page(mm, isolated, result);
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08002493 return;
2494
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002495out_nolock:
2496 trace_mm_collapse_huge_page(mm, isolated, result);
2497 return;
Andrea Arcangelice83d212011-01-13 15:47:06 -08002498out:
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08002499 mem_cgroup_cancel_charge(new_page, memcg, true);
Andrea Arcangelice83d212011-01-13 15:47:06 -08002500 goto out_up_write;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002501}
2502
2503static int khugepaged_scan_pmd(struct mm_struct *mm,
2504 struct vm_area_struct *vma,
2505 unsigned long address,
2506 struct page **hpage)
2507{
Andrea Arcangeliba761492011-01-13 15:46:58 -08002508 pmd_t *pmd;
2509 pte_t *pte, *_pte;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002510 int ret = 0, none_or_zero = 0, result = 0;
2511 struct page *page = NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002512 unsigned long _address;
2513 spinlock_t *ptl;
David Rientjes00ef2d22013-02-22 16:35:36 -08002514 int node = NUMA_NO_NODE;
Ebru Akagunduz10359212015-02-11 15:28:28 -08002515 bool writable = false, referenced = false;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002516
2517 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
2518
Bob Liu62190492012-12-11 16:00:37 -08002519 pmd = mm_find_pmd(mm, address);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002520 if (!pmd) {
2521 result = SCAN_PMD_NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002522 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002523 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002524
Bob Liu9f1b8682013-11-12 15:07:37 -08002525 memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
Andrea Arcangeliba761492011-01-13 15:46:58 -08002526 pte = pte_offset_map_lock(mm, pmd, address, &ptl);
2527 for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR;
2528 _pte++, _address += PAGE_SIZE) {
2529 pte_t pteval = *_pte;
Ebru Akagunduzca0984c2015-04-14 15:45:24 -07002530 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
Andrea Arcangelic1294d02015-09-04 15:46:27 -07002531 if (!userfaultfd_armed(vma) &&
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002532 ++none_or_zero <= khugepaged_max_ptes_none) {
Andrea Arcangeliba761492011-01-13 15:46:58 -08002533 continue;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002534 } else {
2535 result = SCAN_EXCEED_NONE_PTE;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002536 goto out_unmap;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002537 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002538 }
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002539 if (!pte_present(pteval)) {
2540 result = SCAN_PTE_NON_PRESENT;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002541 goto out_unmap;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002542 }
Ebru Akagunduz10359212015-02-11 15:28:28 -08002543 if (pte_write(pteval))
2544 writable = true;
2545
Andrea Arcangeliba761492011-01-13 15:46:58 -08002546 page = vm_normal_page(vma, _address, pteval);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002547 if (unlikely(!page)) {
2548 result = SCAN_PAGE_NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002549 goto out_unmap;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002550 }
Kirill A. Shutemovb1caa952016-01-15 16:52:39 -08002551
2552 /* TODO: teach khugepaged to collapse THP mapped with pte */
2553 if (PageCompound(page)) {
2554 result = SCAN_PAGE_COMPOUND;
2555 goto out_unmap;
2556 }
2557
Andi Kleen5c4b4be2011-03-04 17:36:32 -08002558 /*
Bob Liu9f1b8682013-11-12 15:07:37 -08002559 * Record which node the original page is from and save this
2560 * information to khugepaged_node_load[].
2561 * Khupaged will allocate hugepage from the node has the max
2562 * hit record.
Andi Kleen5c4b4be2011-03-04 17:36:32 -08002563 */
Bob Liu9f1b8682013-11-12 15:07:37 -08002564 node = page_to_nid(page);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002565 if (khugepaged_scan_abort(node)) {
2566 result = SCAN_SCAN_ABORT;
David Rientjes14a4e212014-08-06 16:07:29 -07002567 goto out_unmap;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002568 }
Bob Liu9f1b8682013-11-12 15:07:37 -08002569 khugepaged_node_load[node]++;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002570 if (!PageLRU(page)) {
2571 result = SCAN_SCAN_ABORT;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002572 goto out_unmap;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002573 }
2574 if (PageLocked(page)) {
2575 result = SCAN_PAGE_LOCK;
2576 goto out_unmap;
2577 }
2578 if (!PageAnon(page)) {
2579 result = SCAN_PAGE_ANON;
2580 goto out_unmap;
2581 }
2582
Ebru Akagunduz10359212015-02-11 15:28:28 -08002583 /*
2584 * cannot use mapcount: can't collapse if there's a gup pin.
2585 * The page must only be referenced by the scanned process
2586 * and page swap cache.
2587 */
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002588 if (page_count(page) != 1 + !!PageSwapCache(page)) {
2589 result = SCAN_PAGE_COUNT;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002590 goto out_unmap;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002591 }
Vladimir Davydov33c3fc72015-09-09 15:35:45 -07002592 if (pte_young(pteval) ||
2593 page_is_young(page) || PageReferenced(page) ||
Andrea Arcangeli8ee53822011-01-13 15:47:10 -08002594 mmu_notifier_test_young(vma->vm_mm, address))
Ebru Akagunduz10359212015-02-11 15:28:28 -08002595 referenced = true;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002596 }
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002597 if (writable) {
2598 if (referenced) {
2599 result = SCAN_SUCCEED;
2600 ret = 1;
2601 } else {
2602 result = SCAN_NO_REFERENCED_PAGE;
2603 }
2604 } else {
2605 result = SCAN_PAGE_RO;
2606 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002607out_unmap:
2608 pte_unmap_unlock(pte, ptl);
Bob Liu9f1b8682013-11-12 15:07:37 -08002609 if (ret) {
2610 node = khugepaged_find_target_node();
Andrea Arcangelice83d212011-01-13 15:47:06 -08002611 /* collapse_huge_page will return with the mmap_sem released */
Andi Kleen5c4b4be2011-03-04 17:36:32 -08002612 collapse_huge_page(mm, address, hpage, vma, node);
Bob Liu9f1b8682013-11-12 15:07:37 -08002613 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002614out:
yalin wang16fd0fe2016-01-21 16:40:30 -08002615 trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced,
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002616 none_or_zero, result);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002617 return ret;
2618}
2619
2620static void collect_mm_slot(struct mm_slot *mm_slot)
2621{
2622 struct mm_struct *mm = mm_slot->mm;
2623
Hugh Dickinsb9980cd2012-02-08 17:13:40 -08002624 VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
Andrea Arcangeliba761492011-01-13 15:46:58 -08002625
2626 if (khugepaged_test_exit(mm)) {
2627 /* free mm_slot */
Sasha Levin43b5fbb2013-02-22 16:32:27 -08002628 hash_del(&mm_slot->hash);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002629 list_del(&mm_slot->mm_node);
2630
2631 /*
2632 * Not strictly needed because the mm exited already.
2633 *
2634 * clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
2635 */
2636
2637 /* khugepaged_mm_lock actually not necessary for the below */
2638 free_mm_slot(mm_slot);
2639 mmdrop(mm);
2640 }
2641}
2642
2643static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
2644 struct page **hpage)
H Hartley Sweeten2f1da642011-10-31 17:09:25 -07002645 __releases(&khugepaged_mm_lock)
2646 __acquires(&khugepaged_mm_lock)
Andrea Arcangeliba761492011-01-13 15:46:58 -08002647{
2648 struct mm_slot *mm_slot;
2649 struct mm_struct *mm;
2650 struct vm_area_struct *vma;
2651 int progress = 0;
2652
2653 VM_BUG_ON(!pages);
Hugh Dickinsb9980cd2012-02-08 17:13:40 -08002654 VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
Andrea Arcangeliba761492011-01-13 15:46:58 -08002655
2656 if (khugepaged_scan.mm_slot)
2657 mm_slot = khugepaged_scan.mm_slot;
2658 else {
2659 mm_slot = list_entry(khugepaged_scan.mm_head.next,
2660 struct mm_slot, mm_node);
2661 khugepaged_scan.address = 0;
2662 khugepaged_scan.mm_slot = mm_slot;
2663 }
2664 spin_unlock(&khugepaged_mm_lock);
2665
2666 mm = mm_slot->mm;
2667 down_read(&mm->mmap_sem);
2668 if (unlikely(khugepaged_test_exit(mm)))
2669 vma = NULL;
2670 else
2671 vma = find_vma(mm, khugepaged_scan.address);
2672
2673 progress++;
2674 for (; vma; vma = vma->vm_next) {
2675 unsigned long hstart, hend;
2676
2677 cond_resched();
2678 if (unlikely(khugepaged_test_exit(mm))) {
2679 progress++;
2680 break;
2681 }
Bob Liufa475e52012-12-11 16:00:39 -08002682 if (!hugepage_vma_check(vma)) {
2683skip:
Andrea Arcangeliba761492011-01-13 15:46:58 -08002684 progress++;
2685 continue;
2686 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002687 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
2688 hend = vma->vm_end & HPAGE_PMD_MASK;
Andrea Arcangelia7d6e4e2011-02-15 19:02:45 +01002689 if (hstart >= hend)
2690 goto skip;
2691 if (khugepaged_scan.address > hend)
2692 goto skip;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002693 if (khugepaged_scan.address < hstart)
2694 khugepaged_scan.address = hstart;
Andrea Arcangelia7d6e4e2011-02-15 19:02:45 +01002695 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002696
2697 while (khugepaged_scan.address < hend) {
2698 int ret;
2699 cond_resched();
2700 if (unlikely(khugepaged_test_exit(mm)))
2701 goto breakouterloop;
2702
2703 VM_BUG_ON(khugepaged_scan.address < hstart ||
2704 khugepaged_scan.address + HPAGE_PMD_SIZE >
2705 hend);
2706 ret = khugepaged_scan_pmd(mm, vma,
2707 khugepaged_scan.address,
2708 hpage);
2709 /* move to next address */
2710 khugepaged_scan.address += HPAGE_PMD_SIZE;
2711 progress += HPAGE_PMD_NR;
2712 if (ret)
2713 /* we released mmap_sem so break loop */
2714 goto breakouterloop_mmap_sem;
2715 if (progress >= pages)
2716 goto breakouterloop;
2717 }
2718 }
2719breakouterloop:
2720 up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */
2721breakouterloop_mmap_sem:
2722
2723 spin_lock(&khugepaged_mm_lock);
Andrea Arcangelia7d6e4e2011-02-15 19:02:45 +01002724 VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002725 /*
2726 * Release the current mm_slot if this mm is about to die, or
2727 * if we scanned all vmas of this mm.
2728 */
2729 if (khugepaged_test_exit(mm) || !vma) {
2730 /*
2731 * Make sure that if mm_users is reaching zero while
2732 * khugepaged runs here, khugepaged_exit will find
2733 * mm_slot not pointing to the exiting mm.
2734 */
2735 if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) {
2736 khugepaged_scan.mm_slot = list_entry(
2737 mm_slot->mm_node.next,
2738 struct mm_slot, mm_node);
2739 khugepaged_scan.address = 0;
2740 } else {
2741 khugepaged_scan.mm_slot = NULL;
2742 khugepaged_full_scans++;
2743 }
2744
2745 collect_mm_slot(mm_slot);
2746 }
2747
2748 return progress;
2749}
2750
2751static int khugepaged_has_work(void)
2752{
2753 return !list_empty(&khugepaged_scan.mm_head) &&
2754 khugepaged_enabled();
2755}
2756
2757static int khugepaged_wait_event(void)
2758{
2759 return !list_empty(&khugepaged_scan.mm_head) ||
Xiao Guangrong2017c0b2012-10-08 16:29:44 -07002760 kthread_should_stop();
Andrea Arcangeliba761492011-01-13 15:46:58 -08002761}
2762
Xiao Guangrongd5169042012-10-08 16:29:48 -07002763static void khugepaged_do_scan(void)
2764{
2765 struct page *hpage = NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002766 unsigned int progress = 0, pass_through_head = 0;
2767 unsigned int pages = khugepaged_pages_to_scan;
Xiao Guangrongd5169042012-10-08 16:29:48 -07002768 bool wait = true;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002769
2770 barrier(); /* write khugepaged_pages_to_scan to local stack */
2771
2772 while (progress < pages) {
Xiao Guangrong26234f32012-10-08 16:29:51 -07002773 if (!khugepaged_prealloc_page(&hpage, &wait))
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08002774 break;
Xiao Guangrong26234f32012-10-08 16:29:51 -07002775
Xiao Guangrong420256ef2012-10-08 16:29:49 -07002776 cond_resched();
Andrea Arcangeliba761492011-01-13 15:46:58 -08002777
Jiri Kosinacd092412015-06-24 16:56:07 -07002778 if (unlikely(kthread_should_stop() || try_to_freeze()))
Andrea Arcangeli878aee72011-01-13 15:47:10 -08002779 break;
2780
Andrea Arcangeliba761492011-01-13 15:46:58 -08002781 spin_lock(&khugepaged_mm_lock);
2782 if (!khugepaged_scan.mm_slot)
2783 pass_through_head++;
2784 if (khugepaged_has_work() &&
2785 pass_through_head < 2)
2786 progress += khugepaged_scan_mm_slot(pages - progress,
Xiao Guangrongd5169042012-10-08 16:29:48 -07002787 &hpage);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002788 else
2789 progress = pages;
2790 spin_unlock(&khugepaged_mm_lock);
2791 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002792
Xiao Guangrongd5169042012-10-08 16:29:48 -07002793 if (!IS_ERR_OR_NULL(hpage))
2794 put_page(hpage);
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08002795}
2796
Xiao Guangrong2017c0b2012-10-08 16:29:44 -07002797static void khugepaged_wait_work(void)
2798{
Xiao Guangrong2017c0b2012-10-08 16:29:44 -07002799 if (khugepaged_has_work()) {
2800 if (!khugepaged_scan_sleep_millisecs)
2801 return;
2802
2803 wait_event_freezable_timeout(khugepaged_wait,
2804 kthread_should_stop(),
2805 msecs_to_jiffies(khugepaged_scan_sleep_millisecs));
2806 return;
2807 }
2808
2809 if (khugepaged_enabled())
2810 wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
2811}
2812
Andrea Arcangeliba761492011-01-13 15:46:58 -08002813static int khugepaged(void *none)
2814{
2815 struct mm_slot *mm_slot;
2816
Andrea Arcangeli878aee72011-01-13 15:47:10 -08002817 set_freezable();
Dongsheng Yang8698a742014-03-11 18:09:12 +08002818 set_user_nice(current, MAX_NICE);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002819
Xiao Guangrongb7231782012-10-08 16:29:54 -07002820 while (!kthread_should_stop()) {
2821 khugepaged_do_scan();
2822 khugepaged_wait_work();
2823 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002824
2825 spin_lock(&khugepaged_mm_lock);
2826 mm_slot = khugepaged_scan.mm_slot;
2827 khugepaged_scan.mm_slot = NULL;
2828 if (mm_slot)
2829 collect_mm_slot(mm_slot);
2830 spin_unlock(&khugepaged_mm_lock);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002831 return 0;
2832}
2833
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002834static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
2835 unsigned long haddr, pmd_t *pmd)
2836{
2837 struct mm_struct *mm = vma->vm_mm;
2838 pgtable_t pgtable;
2839 pmd_t _pmd;
2840 int i;
2841
2842 /* leave pmd empty until pte is filled */
2843 pmdp_huge_clear_flush_notify(vma, haddr, pmd);
2844
2845 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2846 pmd_populate(mm, &_pmd, pgtable);
2847
2848 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
2849 pte_t *pte, entry;
2850 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
2851 entry = pte_mkspecial(entry);
2852 pte = pte_offset_map(&_pmd, haddr);
2853 VM_BUG_ON(!pte_none(*pte));
2854 set_pte_at(mm, haddr, pte, entry);
2855 pte_unmap(pte);
2856 }
2857 smp_wmb(); /* make pte visible before pmd */
2858 pmd_populate(mm, pmd, pgtable);
2859 put_huge_zero_page();
2860}
2861
2862static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
Kirill A. Shutemovba988282016-01-15 16:53:56 -08002863 unsigned long haddr, bool freeze)
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002864{
2865 struct mm_struct *mm = vma->vm_mm;
2866 struct page *page;
2867 pgtable_t pgtable;
2868 pmd_t _pmd;
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08002869 bool young, write, dirty;
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03002870 unsigned long addr;
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002871 int i;
2872
2873 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
2874 VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
2875 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
Dan Williams5c7fb562016-01-15 16:56:52 -08002876 VM_BUG_ON(!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd));
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002877
2878 count_vm_event(THP_SPLIT_PMD);
2879
2880 if (vma_is_dax(vma)) {
2881 pmd_t _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);
2882 if (is_huge_zero_pmd(_pmd))
2883 put_huge_zero_page();
2884 return;
2885 } else if (is_huge_zero_pmd(*pmd)) {
2886 return __split_huge_zero_page_pmd(vma, haddr, pmd);
2887 }
2888
2889 page = pmd_page(*pmd);
2890 VM_BUG_ON_PAGE(!page_count(page), page);
2891 atomic_add(HPAGE_PMD_NR - 1, &page->_count);
2892 write = pmd_write(*pmd);
2893 young = pmd_young(*pmd);
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08002894 dirty = pmd_dirty(*pmd);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002895
Aneesh Kumar K.Vc777e2a2016-02-09 06:50:31 +05302896 pmdp_huge_split_prepare(vma, haddr, pmd);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002897 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2898 pmd_populate(mm, &_pmd, pgtable);
2899
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03002900 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002901 pte_t entry, *pte;
2902 /*
2903 * Note that NUMA hinting access restrictions are not
2904 * transferred to avoid any possibility of altering
2905 * permissions across VMAs.
2906 */
Kirill A. Shutemovba988282016-01-15 16:53:56 -08002907 if (freeze) {
2908 swp_entry_t swp_entry;
2909 swp_entry = make_migration_entry(page + i, write);
2910 entry = swp_entry_to_pte(swp_entry);
2911 } else {
2912 entry = mk_pte(page + i, vma->vm_page_prot);
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08002913 entry = maybe_mkwrite(entry, vma);
Kirill A. Shutemovba988282016-01-15 16:53:56 -08002914 if (!write)
2915 entry = pte_wrprotect(entry);
2916 if (!young)
2917 entry = pte_mkold(entry);
2918 }
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08002919 if (dirty)
2920 SetPageDirty(page + i);
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03002921 pte = pte_offset_map(&_pmd, addr);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002922 BUG_ON(!pte_none(*pte));
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03002923 set_pte_at(mm, addr, pte, entry);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002924 atomic_inc(&page[i]._mapcount);
2925 pte_unmap(pte);
2926 }
2927
2928 /*
2929 * Set PG_double_map before dropping compound_mapcount to avoid
2930 * false-negative page_mapped().
2931 */
2932 if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) {
2933 for (i = 0; i < HPAGE_PMD_NR; i++)
2934 atomic_inc(&page[i]._mapcount);
2935 }
2936
2937 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) {
2938 /* Last compound_mapcount is gone. */
2939 __dec_zone_page_state(page, NR_ANON_TRANSPARENT_HUGEPAGES);
2940 if (TestClearPageDoubleMap(page)) {
2941 /* No need in mapcount reference anymore */
2942 for (i = 0; i < HPAGE_PMD_NR; i++)
2943 atomic_dec(&page[i]._mapcount);
2944 }
2945 }
2946
2947 smp_wmb(); /* make pte visible before pmd */
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002948 /*
2949 * Up to this point the pmd is present and huge and userland has the
2950 * whole access to the hugepage during the split (which happens in
2951 * place). If we overwrite the pmd with the not-huge version pointing
2952 * to the pte here (which of course we could if all CPUs were bug
2953 * free), userland could trigger a small page size TLB miss on the
2954 * small sized TLB while the hugepage TLB entry is still established in
2955 * the huge TLB. Some CPU doesn't like that.
2956 * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum
2957 * 383 on page 93. Intel should be safe but is also warns that it's
2958 * only safe if the permission and cache attributes of the two entries
2959 * loaded in the two TLB is identical (which should be the case here).
2960 * But it is generally safer to never allow small and huge TLB entries
2961 * for the same virtual address to be loaded simultaneously. So instead
2962 * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the
2963 * current pmd notpresent (atomically because here the pmd_trans_huge
2964 * and pmd_trans_splitting must remain set at all times on the pmd
2965 * until the split is complete for this pmd), then we flush the SMP TLB
2966 * and finally we write the non-huge version of the pmd entry with
2967 * pmd_populate.
2968 */
2969 pmdp_invalidate(vma, haddr, pmd);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002970 pmd_populate(mm, pmd, pgtable);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002971
2972 if (freeze) {
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03002973 for (i = 0; i < HPAGE_PMD_NR; i++) {
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08002974 page_remove_rmap(page + i, false);
2975 put_page(page + i);
2976 }
2977 }
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002978}
2979
2980void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
2981 unsigned long address)
2982{
2983 spinlock_t *ptl;
2984 struct mm_struct *mm = vma->vm_mm;
Kirill A. Shutemove90309c2016-01-15 16:54:33 -08002985 struct page *page = NULL;
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002986 unsigned long haddr = address & HPAGE_PMD_MASK;
2987
2988 mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);
2989 ptl = pmd_lock(mm, pmd);
Dan Williams5c7fb562016-01-15 16:56:52 -08002990 if (pmd_trans_huge(*pmd)) {
2991 page = pmd_page(*pmd);
2992 if (PageMlocked(page))
2993 get_page(page);
2994 else
2995 page = NULL;
2996 } else if (!pmd_devmap(*pmd))
Kirill A. Shutemove90309c2016-01-15 16:54:33 -08002997 goto out;
Kirill A. Shutemove90309c2016-01-15 16:54:33 -08002998 __split_huge_pmd_locked(vma, pmd, haddr, false);
Kirill A. Shutemove90309c2016-01-15 16:54:33 -08002999out:
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003000 spin_unlock(ptl);
3001 mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE);
Kirill A. Shutemove90309c2016-01-15 16:54:33 -08003002 if (page) {
3003 lock_page(page);
3004 munlock_vma_page(page);
3005 unlock_page(page);
3006 put_page(page);
3007 }
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003008}
3009
Kirill A. Shutemov78ddc532016-01-15 16:52:42 -08003010static void split_huge_pmd_address(struct vm_area_struct *vma,
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003011 unsigned long address)
3012{
Hugh Dickinsf72e7dc2014-06-23 13:22:05 -07003013 pgd_t *pgd;
3014 pud_t *pud;
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003015 pmd_t *pmd;
3016
3017 VM_BUG_ON(!(address & ~HPAGE_PMD_MASK));
3018
Kirill A. Shutemov78ddc532016-01-15 16:52:42 -08003019 pgd = pgd_offset(vma->vm_mm, address);
Hugh Dickinsf72e7dc2014-06-23 13:22:05 -07003020 if (!pgd_present(*pgd))
3021 return;
3022
3023 pud = pud_offset(pgd, address);
3024 if (!pud_present(*pud))
3025 return;
3026
3027 pmd = pmd_offset(pud, address);
Dan Williams5c7fb562016-01-15 16:56:52 -08003028 if (!pmd_present(*pmd) || (!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd)))
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003029 return;
3030 /*
3031 * Caller holds the mmap_sem write mode, so a huge pmd cannot
3032 * materialize from under us.
3033 */
Kirill A. Shutemovad0bed22016-01-15 16:52:53 -08003034 split_huge_pmd(vma, pmd, address);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003035}
3036
Kirill A. Shutemove1b99962015-09-08 14:58:37 -07003037void vma_adjust_trans_huge(struct vm_area_struct *vma,
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003038 unsigned long start,
3039 unsigned long end,
3040 long adjust_next)
3041{
3042 /*
3043 * If the new start address isn't hpage aligned and it could
3044 * previously contain an hugepage: check if we need to split
3045 * an huge pmd.
3046 */
3047 if (start & ~HPAGE_PMD_MASK &&
3048 (start & HPAGE_PMD_MASK) >= vma->vm_start &&
3049 (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
Kirill A. Shutemov78ddc532016-01-15 16:52:42 -08003050 split_huge_pmd_address(vma, start);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003051
3052 /*
3053 * If the new end address isn't hpage aligned and it could
3054 * previously contain an hugepage: check if we need to split
3055 * an huge pmd.
3056 */
3057 if (end & ~HPAGE_PMD_MASK &&
3058 (end & HPAGE_PMD_MASK) >= vma->vm_start &&
3059 (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
Kirill A. Shutemov78ddc532016-01-15 16:52:42 -08003060 split_huge_pmd_address(vma, end);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003061
3062 /*
3063 * If we're also updating the vma->vm_next->vm_start, if the new
3064 * vm_next->vm_start isn't page aligned and it could previously
3065 * contain an hugepage: check if we need to split an huge pmd.
3066 */
3067 if (adjust_next > 0) {
3068 struct vm_area_struct *next = vma->vm_next;
3069 unsigned long nstart = next->vm_start;
3070 nstart += adjust_next << PAGE_SHIFT;
3071 if (nstart & ~HPAGE_PMD_MASK &&
3072 (nstart & HPAGE_PMD_MASK) >= next->vm_start &&
3073 (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)
Kirill A. Shutemov78ddc532016-01-15 16:52:42 -08003074 split_huge_pmd_address(next, nstart);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003075 }
3076}
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003077
3078static void freeze_page_vma(struct vm_area_struct *vma, struct page *page,
3079 unsigned long address)
3080{
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003081 unsigned long haddr = address & HPAGE_PMD_MASK;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003082 spinlock_t *ptl;
3083 pgd_t *pgd;
3084 pud_t *pud;
3085 pmd_t *pmd;
3086 pte_t *pte;
3087 int i, nr = HPAGE_PMD_NR;
3088
3089 /* Skip pages which doesn't belong to the VMA */
3090 if (address < vma->vm_start) {
3091 int off = (vma->vm_start - address) >> PAGE_SHIFT;
3092 page += off;
3093 nr -= off;
3094 address = vma->vm_start;
3095 }
3096
3097 pgd = pgd_offset(vma->vm_mm, address);
3098 if (!pgd_present(*pgd))
3099 return;
3100 pud = pud_offset(pgd, address);
3101 if (!pud_present(*pud))
3102 return;
3103 pmd = pmd_offset(pud, address);
3104 ptl = pmd_lock(vma->vm_mm, pmd);
3105 if (!pmd_present(*pmd)) {
3106 spin_unlock(ptl);
3107 return;
3108 }
3109 if (pmd_trans_huge(*pmd)) {
3110 if (page == pmd_page(*pmd))
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003111 __split_huge_pmd_locked(vma, pmd, haddr, true);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003112 spin_unlock(ptl);
3113 return;
3114 }
3115 spin_unlock(ptl);
3116
3117 pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl);
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003118 for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) {
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003119 pte_t entry, swp_pte;
3120 swp_entry_t swp_entry;
3121
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003122 /*
3123 * We've just crossed page table boundary: need to map next one.
3124 * It can happen if THP was mremaped to non PMD-aligned address.
3125 */
3126 if (unlikely(address == haddr + HPAGE_PMD_SIZE)) {
3127 pte_unmap_unlock(pte - 1, ptl);
3128 pmd = mm_find_pmd(vma->vm_mm, address);
3129 if (!pmd)
3130 return;
3131 pte = pte_offset_map_lock(vma->vm_mm, pmd,
3132 address, &ptl);
3133 }
3134
3135 if (!pte_present(*pte))
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003136 continue;
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003137 if (page_to_pfn(page) != pte_pfn(*pte))
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003138 continue;
3139 flush_cache_page(vma, address, page_to_pfn(page));
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003140 entry = ptep_clear_flush(vma, address, pte);
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08003141 if (pte_dirty(entry))
3142 SetPageDirty(page);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003143 swp_entry = make_migration_entry(page, pte_write(entry));
3144 swp_pte = swp_entry_to_pte(swp_entry);
3145 if (pte_soft_dirty(entry))
3146 swp_pte = pte_swp_mksoft_dirty(swp_pte);
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003147 set_pte_at(vma->vm_mm, address, pte, swp_pte);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003148 page_remove_rmap(page, false);
3149 put_page(page);
3150 }
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003151 pte_unmap_unlock(pte - 1, ptl);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003152}
3153
3154static void freeze_page(struct anon_vma *anon_vma, struct page *page)
3155{
3156 struct anon_vma_chain *avc;
3157 pgoff_t pgoff = page_to_pgoff(page);
3158
3159 VM_BUG_ON_PAGE(!PageHead(page), page);
3160
3161 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root, pgoff,
3162 pgoff + HPAGE_PMD_NR - 1) {
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003163 unsigned long address = __vma_address(page, avc->vma);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003164
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003165 mmu_notifier_invalidate_range_start(avc->vma->vm_mm,
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003166 address, address + HPAGE_PMD_SIZE);
3167 freeze_page_vma(avc->vma, page, address);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003168 mmu_notifier_invalidate_range_end(avc->vma->vm_mm,
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003169 address, address + HPAGE_PMD_SIZE);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003170 }
3171}
3172
3173static void unfreeze_page_vma(struct vm_area_struct *vma, struct page *page,
3174 unsigned long address)
3175{
3176 spinlock_t *ptl;
3177 pmd_t *pmd;
3178 pte_t *pte, entry;
3179 swp_entry_t swp_entry;
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003180 unsigned long haddr = address & HPAGE_PMD_MASK;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003181 int i, nr = HPAGE_PMD_NR;
3182
3183 /* Skip pages which doesn't belong to the VMA */
3184 if (address < vma->vm_start) {
3185 int off = (vma->vm_start - address) >> PAGE_SHIFT;
3186 page += off;
3187 nr -= off;
3188 address = vma->vm_start;
3189 }
3190
3191 pmd = mm_find_pmd(vma->vm_mm, address);
3192 if (!pmd)
3193 return;
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003194
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003195 pte = pte_offset_map_lock(vma->vm_mm, pmd, address, &ptl);
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003196 for (i = 0; i < nr; i++, address += PAGE_SIZE, page++, pte++) {
3197 /*
3198 * We've just crossed page table boundary: need to map next one.
3199 * It can happen if THP was mremaped to non-PMD aligned address.
3200 */
3201 if (unlikely(address == haddr + HPAGE_PMD_SIZE)) {
3202 pte_unmap_unlock(pte - 1, ptl);
3203 pmd = mm_find_pmd(vma->vm_mm, address);
3204 if (!pmd)
3205 return;
3206 pte = pte_offset_map_lock(vma->vm_mm, pmd,
3207 address, &ptl);
3208 }
3209
3210 if (!is_swap_pte(*pte))
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003211 continue;
3212
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003213 swp_entry = pte_to_swp_entry(*pte);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003214 if (!is_migration_entry(swp_entry))
3215 continue;
3216 if (migration_entry_to_page(swp_entry) != page)
3217 continue;
3218
3219 get_page(page);
3220 page_add_anon_rmap(page, vma, address, false);
3221
3222 entry = pte_mkold(mk_pte(page, vma->vm_page_prot));
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08003223 if (PageDirty(page))
3224 entry = pte_mkdirty(entry);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003225 if (is_write_migration_entry(swp_entry))
3226 entry = maybe_mkwrite(entry, vma);
3227
3228 flush_dcache_page(page);
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003229 set_pte_at(vma->vm_mm, address, pte, entry);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003230
3231 /* No need to invalidate - it was non-present before */
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003232 update_mmu_cache(vma, address, pte);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003233 }
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003234 pte_unmap_unlock(pte - 1, ptl);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003235}
3236
3237static void unfreeze_page(struct anon_vma *anon_vma, struct page *page)
3238{
3239 struct anon_vma_chain *avc;
3240 pgoff_t pgoff = page_to_pgoff(page);
3241
3242 anon_vma_interval_tree_foreach(avc, &anon_vma->rb_root,
3243 pgoff, pgoff + HPAGE_PMD_NR - 1) {
3244 unsigned long address = __vma_address(page, avc->vma);
3245
3246 mmu_notifier_invalidate_range_start(avc->vma->vm_mm,
3247 address, address + HPAGE_PMD_SIZE);
3248 unfreeze_page_vma(avc->vma, page, address);
3249 mmu_notifier_invalidate_range_end(avc->vma->vm_mm,
3250 address, address + HPAGE_PMD_SIZE);
3251 }
3252}
3253
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07003254static void __split_huge_page_tail(struct page *head, int tail,
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003255 struct lruvec *lruvec, struct list_head *list)
3256{
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003257 struct page *page_tail = head + tail;
3258
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07003259 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003260 VM_BUG_ON_PAGE(atomic_read(&page_tail->_count) != 0, page_tail);
3261
3262 /*
3263 * tail_page->_count is zero and not changing from under us. But
3264 * get_page_unless_zero() may be running from under us on the
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07003265 * tail_page. If we used atomic_set() below instead of atomic_inc(), we
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003266 * would then run atomic_set() concurrently with
3267 * get_page_unless_zero(), and atomic_set() is implemented in C not
3268 * using locked ops. spin_unlock on x86 sometime uses locked ops
3269 * because of PPro errata 66, 92, so unless somebody can guarantee
3270 * atomic_set() here would be safe on all archs (and not only on x86),
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07003271 * it's safer to use atomic_inc().
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003272 */
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07003273 atomic_inc(&page_tail->_count);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003274
3275 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
3276 page_tail->flags |= (head->flags &
3277 ((1L << PG_referenced) |
3278 (1L << PG_swapbacked) |
3279 (1L << PG_mlocked) |
3280 (1L << PG_uptodate) |
3281 (1L << PG_active) |
3282 (1L << PG_locked) |
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08003283 (1L << PG_unevictable) |
3284 (1L << PG_dirty)));
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003285
3286 /*
3287 * After clearing PageTail the gup refcount can be released.
3288 * Page flags also must be visible before we make the page non-compound.
3289 */
3290 smp_wmb();
3291
3292 clear_compound_head(page_tail);
3293
3294 if (page_is_young(head))
3295 set_page_young(page_tail);
3296 if (page_is_idle(head))
3297 set_page_idle(page_tail);
3298
3299 /* ->mapping in first tail page is compound_mapcount */
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003300 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003301 page_tail);
3302 page_tail->mapping = head->mapping;
3303
3304 page_tail->index = head->index + tail;
3305 page_cpupid_xchg_last(page_tail, page_cpupid_last(head));
3306 lru_add_page_tail(head, page_tail, lruvec, list);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003307}
3308
3309static void __split_huge_page(struct page *page, struct list_head *list)
3310{
3311 struct page *head = compound_head(page);
3312 struct zone *zone = page_zone(head);
3313 struct lruvec *lruvec;
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07003314 int i;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003315
3316 /* prevent PageLRU to go away from under us, and freeze lru stats */
3317 spin_lock_irq(&zone->lru_lock);
3318 lruvec = mem_cgroup_page_lruvec(head, zone);
3319
3320 /* complete memcg works before add pages to LRU */
3321 mem_cgroup_split_huge_fixup(head);
3322
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003323 for (i = HPAGE_PMD_NR - 1; i >= 1; i--)
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07003324 __split_huge_page_tail(head, i, lruvec, list);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003325
3326 ClearPageCompound(head);
3327 spin_unlock_irq(&zone->lru_lock);
3328
3329 unfreeze_page(page_anon_vma(head), head);
3330
3331 for (i = 0; i < HPAGE_PMD_NR; i++) {
3332 struct page *subpage = head + i;
3333 if (subpage == page)
3334 continue;
3335 unlock_page(subpage);
3336
3337 /*
3338 * Subpages may be freed if there wasn't any mapping
3339 * like if add_to_swap() is running on a lru page that
3340 * had its mapping zapped. And freeing these pages
3341 * requires taking the lru_lock so we do the put_page
3342 * of the tail pages after the split is complete.
3343 */
3344 put_page(subpage);
3345 }
3346}
3347
Kirill A. Shutemovb20ce5e2016-01-15 16:54:37 -08003348int total_mapcount(struct page *page)
3349{
3350 int i, ret;
3351
3352 VM_BUG_ON_PAGE(PageTail(page), page);
3353
3354 if (likely(!PageCompound(page)))
3355 return atomic_read(&page->_mapcount) + 1;
3356
3357 ret = compound_mapcount(page);
3358 if (PageHuge(page))
3359 return ret;
3360 for (i = 0; i < HPAGE_PMD_NR; i++)
3361 ret += atomic_read(&page[i]._mapcount) + 1;
3362 if (PageDoubleMap(page))
3363 ret -= HPAGE_PMD_NR;
3364 return ret;
3365}
3366
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003367/*
3368 * This function splits huge page into normal pages. @page can point to any
3369 * subpage of huge page to split. Split doesn't change the position of @page.
3370 *
3371 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
3372 * The huge page must be locked.
3373 *
3374 * If @list is null, tail pages will be added to LRU list, otherwise, to @list.
3375 *
3376 * Both head page and tail pages will inherit mapping, flags, and so on from
3377 * the hugepage.
3378 *
3379 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if
3380 * they are not mapped.
3381 *
3382 * Returns 0 if the hugepage is split successfully.
3383 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
3384 * us.
3385 */
3386int split_huge_page_to_list(struct page *page, struct list_head *list)
3387{
3388 struct page *head = compound_head(page);
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003389 struct pglist_data *pgdata = NODE_DATA(page_to_nid(head));
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003390 struct anon_vma *anon_vma;
3391 int count, mapcount, ret;
Kirill A. Shutemovd9654322016-01-15 16:54:43 -08003392 bool mlocked;
Kirill A. Shutemov0b9b6ff2016-01-20 14:58:09 -08003393 unsigned long flags;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003394
3395 VM_BUG_ON_PAGE(is_huge_zero_page(page), page);
3396 VM_BUG_ON_PAGE(!PageAnon(page), page);
3397 VM_BUG_ON_PAGE(!PageLocked(page), page);
3398 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
3399 VM_BUG_ON_PAGE(!PageCompound(page), page);
3400
3401 /*
3402 * The caller does not necessarily hold an mmap_sem that would prevent
3403 * the anon_vma disappearing so we first we take a reference to it
3404 * and then lock the anon_vma for write. This is similar to
3405 * page_lock_anon_vma_read except the write lock is taken to serialise
3406 * against parallel split or collapse operations.
3407 */
3408 anon_vma = page_get_anon_vma(head);
3409 if (!anon_vma) {
3410 ret = -EBUSY;
3411 goto out;
3412 }
3413 anon_vma_lock_write(anon_vma);
3414
3415 /*
3416 * Racy check if we can split the page, before freeze_page() will
3417 * split PMDs
3418 */
3419 if (total_mapcount(head) != page_count(head) - 1) {
3420 ret = -EBUSY;
3421 goto out_unlock;
3422 }
3423
Kirill A. Shutemovd9654322016-01-15 16:54:43 -08003424 mlocked = PageMlocked(page);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003425 freeze_page(anon_vma, head);
3426 VM_BUG_ON_PAGE(compound_mapcount(head), head);
3427
Kirill A. Shutemovd9654322016-01-15 16:54:43 -08003428 /* Make sure the page is not on per-CPU pagevec as it takes pin */
3429 if (mlocked)
3430 lru_add_drain();
3431
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003432 /* Prevent deferred_split_scan() touching ->_count */
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003433 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003434 count = page_count(head);
3435 mapcount = total_mapcount(head);
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003436 if (!mapcount && count == 1) {
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003437 if (!list_empty(page_deferred_list(head))) {
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003438 pgdata->split_queue_len--;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003439 list_del(page_deferred_list(head));
3440 }
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003441 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003442 __split_huge_page(page, list);
3443 ret = 0;
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003444 } else if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) {
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003445 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003446 pr_alert("total_mapcount: %u, page_count(): %u\n",
3447 mapcount, count);
3448 if (PageTail(page))
3449 dump_page(head, NULL);
Kirill A. Shutemovbd560862016-01-15 16:55:46 -08003450 dump_page(page, "total_mapcount(head) > 0");
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003451 BUG();
3452 } else {
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003453 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003454 unfreeze_page(anon_vma, head);
3455 ret = -EBUSY;
3456 }
3457
3458out_unlock:
3459 anon_vma_unlock_write(anon_vma);
3460 put_anon_vma(anon_vma);
3461out:
3462 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
3463 return ret;
3464}
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003465
3466void free_transhuge_page(struct page *page)
3467{
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003468 struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003469 unsigned long flags;
3470
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003471 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003472 if (!list_empty(page_deferred_list(page))) {
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003473 pgdata->split_queue_len--;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003474 list_del(page_deferred_list(page));
3475 }
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003476 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003477 free_compound_page(page);
3478}
3479
3480void deferred_split_huge_page(struct page *page)
3481{
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003482 struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003483 unsigned long flags;
3484
3485 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
3486
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003487 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003488 if (list_empty(page_deferred_list(page))) {
Kirill A. Shutemovf9719a02016-03-17 14:18:45 -07003489 count_vm_event(THP_DEFERRED_SPLIT_PAGE);
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003490 list_add_tail(page_deferred_list(page), &pgdata->split_queue);
3491 pgdata->split_queue_len++;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003492 }
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003493 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003494}
3495
3496static unsigned long deferred_split_count(struct shrinker *shrink,
3497 struct shrink_control *sc)
3498{
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003499 struct pglist_data *pgdata = NODE_DATA(sc->nid);
Kirill A. Shutemovcb8d68e2016-02-02 16:57:12 -08003500 return ACCESS_ONCE(pgdata->split_queue_len);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003501}
3502
3503static unsigned long deferred_split_scan(struct shrinker *shrink,
3504 struct shrink_control *sc)
3505{
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003506 struct pglist_data *pgdata = NODE_DATA(sc->nid);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003507 unsigned long flags;
3508 LIST_HEAD(list), *pos, *next;
3509 struct page *page;
3510 int split = 0;
3511
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003512 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003513 /* Take pin on all head pages to avoid freeing them under us */
Kirill A. Shutemovae026202016-02-05 15:36:53 -08003514 list_for_each_safe(pos, next, &pgdata->split_queue) {
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003515 page = list_entry((void *)pos, struct page, mapping);
3516 page = compound_head(page);
Kirill A. Shutemove3ae1952016-02-02 16:57:15 -08003517 if (get_page_unless_zero(page)) {
3518 list_move(page_deferred_list(page), &list);
3519 } else {
3520 /* We lost race with put_compound_page() */
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003521 list_del_init(page_deferred_list(page));
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003522 pgdata->split_queue_len--;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003523 }
Kirill A. Shutemove3ae1952016-02-02 16:57:15 -08003524 if (!--sc->nr_to_scan)
3525 break;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003526 }
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003527 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003528
3529 list_for_each_safe(pos, next, &list) {
3530 page = list_entry((void *)pos, struct page, mapping);
3531 lock_page(page);
3532 /* split_huge_page() removes page from list on success */
3533 if (!split_huge_page(page))
3534 split++;
3535 unlock_page(page);
3536 put_page(page);
3537 }
3538
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003539 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
3540 list_splice_tail(&list, &pgdata->split_queue);
3541 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003542
Kirill A. Shutemovcb8d68e2016-02-02 16:57:12 -08003543 /*
3544 * Stop shrinker if we didn't split any page, but the queue is empty.
3545 * This can happen if pages were freed under us.
3546 */
3547 if (!split && list_empty(&pgdata->split_queue))
3548 return SHRINK_STOP;
3549 return split;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003550}
3551
3552static struct shrinker deferred_split_shrinker = {
3553 .count_objects = deferred_split_count,
3554 .scan_objects = deferred_split_scan,
3555 .seeks = DEFAULT_SEEKS,
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003556 .flags = SHRINKER_NUMA_AWARE,
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003557};
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08003558
3559#ifdef CONFIG_DEBUG_FS
3560static int split_huge_pages_set(void *data, u64 val)
3561{
3562 struct zone *zone;
3563 struct page *page;
3564 unsigned long pfn, max_zone_pfn;
3565 unsigned long total = 0, split = 0;
3566
3567 if (val != 1)
3568 return -EINVAL;
3569
3570 for_each_populated_zone(zone) {
3571 max_zone_pfn = zone_end_pfn(zone);
3572 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
3573 if (!pfn_valid(pfn))
3574 continue;
3575
3576 page = pfn_to_page(pfn);
3577 if (!get_page_unless_zero(page))
3578 continue;
3579
3580 if (zone != page_zone(page))
3581 goto next;
3582
3583 if (!PageHead(page) || !PageAnon(page) ||
3584 PageHuge(page))
3585 goto next;
3586
3587 total++;
3588 lock_page(page);
3589 if (!split_huge_page(page))
3590 split++;
3591 unlock_page(page);
3592next:
3593 put_page(page);
3594 }
3595 }
3596
3597 pr_info("%lu of %lu THP split", split, total);
3598
3599 return 0;
3600}
3601DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set,
3602 "%llu\n");
3603
3604static int __init split_huge_pages_debugfs(void)
3605{
3606 void *ret;
3607
3608 ret = debugfs_create_file("split_huge_pages", 0644, NULL, NULL,
3609 &split_huge_pages_fops);
3610 if (!ret)
3611 pr_warn("Failed to create split_huge_pages in debugfs");
3612 return 0;
3613}
3614late_initcall(split_huge_pages_debugfs);
3615#endif