blob: 6c524984931b9725d28ce3106efc38dadf01d548 [file] [log] [blame]
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001/*
2 * Copyright (C) 2009 Red Hat, Inc.
3 *
4 * This work is licensed under the terms of the GNU GPL, version 2. See
5 * the COPYING file in the top-level directory.
6 */
7
Andrew Mortonae3a8c12014-06-04 16:06:58 -07008#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -080010#include <linux/mm.h>
11#include <linux/sched.h>
12#include <linux/highmem.h>
13#include <linux/hugetlb.h>
14#include <linux/mmu_notifier.h>
15#include <linux/rmap.h>
16#include <linux/swap.h>
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -080017#include <linux/shrinker.h>
Andrea Arcangeliba761492011-01-13 15:46:58 -080018#include <linux/mm_inline.h>
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -080019#include <linux/swapops.h>
Matthew Wilcox4897c762015-09-08 14:58:45 -070020#include <linux/dax.h>
Andrea Arcangeliba761492011-01-13 15:46:58 -080021#include <linux/kthread.h>
22#include <linux/khugepaged.h>
Andrea Arcangeli878aee72011-01-13 15:47:10 -080023#include <linux/freezer.h>
Dan Williamsf25748e32016-01-15 16:56:43 -080024#include <linux/pfn_t.h>
Andrea Arcangelia664b2d2011-01-13 15:47:17 -080025#include <linux/mman.h>
Dan Williams3565fce2016-01-15 16:56:55 -080026#include <linux/memremap.h>
Ralf Baechle325adeb2012-10-15 13:44:56 +020027#include <linux/pagemap.h>
Kirill A. Shutemov49071d42016-01-15 16:54:40 -080028#include <linux/debugfs.h>
Mel Gorman4daae3b2012-11-02 11:33:45 +000029#include <linux/migrate.h>
Sasha Levin43b5fbb2013-02-22 16:32:27 -080030#include <linux/hashtable.h>
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -070031#include <linux/userfaultfd_k.h>
Vladimir Davydov33c3fc72015-09-09 15:35:45 -070032#include <linux/page_idle.h>
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -070033#include <linux/shmem_fs.h>
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -080034
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -080035#include <asm/tlb.h>
36#include <asm/pgalloc.h>
37#include "internal.h"
38
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -080039enum scan_result {
40 SCAN_FAIL,
41 SCAN_SUCCEED,
42 SCAN_PMD_NULL,
43 SCAN_EXCEED_NONE_PTE,
44 SCAN_PTE_NON_PRESENT,
45 SCAN_PAGE_RO,
46 SCAN_NO_REFERENCED_PAGE,
47 SCAN_PAGE_NULL,
48 SCAN_SCAN_ABORT,
49 SCAN_PAGE_COUNT,
50 SCAN_PAGE_LRU,
51 SCAN_PAGE_LOCK,
52 SCAN_PAGE_ANON,
Kirill A. Shutemovb1caa952016-01-15 16:52:39 -080053 SCAN_PAGE_COMPOUND,
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -080054 SCAN_ANY_PROCESS,
55 SCAN_VMA_NULL,
56 SCAN_VMA_CHECK,
57 SCAN_ADDRESS_RANGE,
58 SCAN_SWAP_CACHE_PAGE,
59 SCAN_DEL_PAGE_LRU,
60 SCAN_ALLOC_HUGE_PAGE_FAIL,
Ebru Akagunduz70652f62016-07-26 15:24:59 -070061 SCAN_CGROUP_CHARGE_FAIL,
62 SCAN_EXCEED_SWAP_PTE
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -080063};
64
65#define CREATE_TRACE_POINTS
66#include <trace/events/huge_memory.h>
67
Andrea Arcangeliba761492011-01-13 15:46:58 -080068/*
Jianguo Wu8bfa3f92013-11-12 15:07:16 -080069 * By default transparent hugepage support is disabled in order that avoid
70 * to risk increase the memory footprint of applications without a guaranteed
71 * benefit. When transparent hugepage support is enabled, is for all mappings,
72 * and khugepaged scans all mappings.
73 * Defrag is invoked by khugepaged hugepage allocations and by page faults
74 * for all hugepage allocations.
Andrea Arcangeliba761492011-01-13 15:46:58 -080075 */
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -080076unsigned long transparent_hugepage_flags __read_mostly =
Andrea Arcangeli13ece882011-01-13 15:47:07 -080077#ifdef CONFIG_TRANSPARENT_HUGEPAGE_ALWAYS
Andrea Arcangeliba761492011-01-13 15:46:58 -080078 (1<<TRANSPARENT_HUGEPAGE_FLAG)|
Andrea Arcangeli13ece882011-01-13 15:47:07 -080079#endif
80#ifdef CONFIG_TRANSPARENT_HUGEPAGE_MADVISE
81 (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG)|
82#endif
Mel Gorman444eb2a42016-03-17 14:19:23 -070083 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG)|
Kirill A. Shutemov79da5402012-12-12 13:51:12 -080084 (1<<TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG)|
85 (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
Andrea Arcangeliba761492011-01-13 15:46:58 -080086
87/* default scan 8*512 pte (or vmas) every 30 second */
Kirill A. Shutemovff20c2e2016-03-01 09:45:14 +053088static unsigned int khugepaged_pages_to_scan __read_mostly;
Andrea Arcangeliba761492011-01-13 15:46:58 -080089static unsigned int khugepaged_pages_collapsed;
90static unsigned int khugepaged_full_scans;
91static unsigned int khugepaged_scan_sleep_millisecs __read_mostly = 10000;
92/* during fragmentation poll the hugepage allocator once every minute */
93static unsigned int khugepaged_alloc_sleep_millisecs __read_mostly = 60000;
David Rientjesf0508972016-05-20 16:58:56 -070094static unsigned long khugepaged_sleep_expire;
Andrea Arcangeliba761492011-01-13 15:46:58 -080095static struct task_struct *khugepaged_thread __read_mostly;
96static DEFINE_MUTEX(khugepaged_mutex);
97static DEFINE_SPINLOCK(khugepaged_mm_lock);
98static DECLARE_WAIT_QUEUE_HEAD(khugepaged_wait);
99/*
100 * default collapse hugepages if there is at least one pte mapped like
101 * it would have happened if the vma was large enough during page
102 * fault.
103 */
Kirill A. Shutemovff20c2e2016-03-01 09:45:14 +0530104static unsigned int khugepaged_max_ptes_none __read_mostly;
Ebru Akagunduz70652f62016-07-26 15:24:59 -0700105static unsigned int khugepaged_max_ptes_swap __read_mostly;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800106
107static int khugepaged(void *none);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800108static int khugepaged_slab_init(void);
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700109static void khugepaged_slab_exit(void);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800110
Sasha Levin43b5fbb2013-02-22 16:32:27 -0800111#define MM_SLOTS_HASH_BITS 10
112static __read_mostly DEFINE_HASHTABLE(mm_slots_hash, MM_SLOTS_HASH_BITS);
113
Andrea Arcangeliba761492011-01-13 15:46:58 -0800114static struct kmem_cache *mm_slot_cache __read_mostly;
115
116/**
117 * struct mm_slot - hash lookup from mm to mm_slot
118 * @hash: hash collision list
119 * @mm_node: khugepaged scan list headed in khugepaged_scan.mm_head
120 * @mm: the mm that this information is valid for
121 */
122struct mm_slot {
123 struct hlist_node hash;
124 struct list_head mm_node;
125 struct mm_struct *mm;
126};
127
128/**
129 * struct khugepaged_scan - cursor for scanning
130 * @mm_head: the head of the mm list to scan
131 * @mm_slot: the current mm_slot we are scanning
132 * @address: the next address inside that to be scanned
133 *
134 * There is only the one khugepaged_scan instance of this cursor structure.
135 */
136struct khugepaged_scan {
137 struct list_head mm_head;
138 struct mm_slot *mm_slot;
139 unsigned long address;
H Hartley Sweeten2f1da642011-10-31 17:09:25 -0700140};
141static struct khugepaged_scan khugepaged_scan = {
Andrea Arcangeliba761492011-01-13 15:46:58 -0800142 .mm_head = LIST_HEAD_INIT(khugepaged_scan.mm_head),
143};
144
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800145static struct shrinker deferred_split_shrinker;
Andrea Arcangelif0005652011-01-13 15:47:04 -0800146
Nicholas Krause2c0b80d2015-09-08 15:00:33 -0700147static void set_recommended_min_free_kbytes(void)
Andrea Arcangelif0005652011-01-13 15:47:04 -0800148{
149 struct zone *zone;
150 int nr_zones = 0;
151 unsigned long recommended_min;
Andrea Arcangelif0005652011-01-13 15:47:04 -0800152
Andrea Arcangelif0005652011-01-13 15:47:04 -0800153 for_each_populated_zone(zone)
154 nr_zones++;
155
Mel Gorman974a7862015-11-06 16:28:34 -0800156 /* Ensure 2 pageblocks are free to assist fragmentation avoidance */
Andrea Arcangelif0005652011-01-13 15:47:04 -0800157 recommended_min = pageblock_nr_pages * nr_zones * 2;
158
159 /*
160 * Make sure that on average at least two pageblocks are almost free
161 * of another type, one for a migratetype to fall back to and a
162 * second to avoid subsequent fallbacks of other types There are 3
163 * MIGRATE_TYPES we care about.
164 */
165 recommended_min += pageblock_nr_pages * nr_zones *
166 MIGRATE_PCPTYPES * MIGRATE_PCPTYPES;
167
168 /* don't ever allow to reserve more than 5% of the lowmem */
169 recommended_min = min(recommended_min,
170 (unsigned long) nr_free_buffer_pages() / 20);
171 recommended_min <<= (PAGE_SHIFT-10);
172
Han Pingtian42aa83c2014-01-23 15:53:28 -0800173 if (recommended_min > min_free_kbytes) {
174 if (user_min_free_kbytes >= 0)
Joe Perches756a0252016-03-17 14:19:47 -0700175 pr_info("raising min_free_kbytes from %d to %lu to help transparent hugepage allocations\n",
Han Pingtian42aa83c2014-01-23 15:53:28 -0800176 min_free_kbytes, recommended_min);
177
Andrea Arcangelif0005652011-01-13 15:47:04 -0800178 min_free_kbytes = recommended_min;
Han Pingtian42aa83c2014-01-23 15:53:28 -0800179 }
Andrea Arcangelif0005652011-01-13 15:47:04 -0800180 setup_per_zone_wmarks();
Andrea Arcangelif0005652011-01-13 15:47:04 -0800181}
Andrea Arcangelif0005652011-01-13 15:47:04 -0800182
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700183static int start_stop_khugepaged(void)
Andrea Arcangeliba761492011-01-13 15:46:58 -0800184{
185 int err = 0;
186 if (khugepaged_enabled()) {
Andrea Arcangeliba761492011-01-13 15:46:58 -0800187 if (!khugepaged_thread)
188 khugepaged_thread = kthread_run(khugepaged, NULL,
189 "khugepaged");
Viresh Kumar18e8e5c2015-08-12 15:59:46 +0530190 if (IS_ERR(khugepaged_thread)) {
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700191 pr_err("khugepaged: kthread_run(khugepaged) failed\n");
Andrea Arcangeliba761492011-01-13 15:46:58 -0800192 err = PTR_ERR(khugepaged_thread);
193 khugepaged_thread = NULL;
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700194 goto fail;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800195 }
Xiao Guangrong911891a2012-10-08 16:29:41 -0700196
197 if (!list_empty(&khugepaged_scan.mm_head))
Andrea Arcangeliba761492011-01-13 15:46:58 -0800198 wake_up_interruptible(&khugepaged_wait);
Andrea Arcangelif0005652011-01-13 15:47:04 -0800199
200 set_recommended_min_free_kbytes();
Xiao Guangrong911891a2012-10-08 16:29:41 -0700201 } else if (khugepaged_thread) {
Xiao Guangrong911891a2012-10-08 16:29:41 -0700202 kthread_stop(khugepaged_thread);
203 khugepaged_thread = NULL;
204 }
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700205fail:
Andrea Arcangeliba761492011-01-13 15:46:58 -0800206 return err;
207}
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800208
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800209static atomic_t huge_zero_refcount;
Wang, Yalin56873f42015-02-11 15:24:51 -0800210struct page *huge_zero_page __read_mostly;
Kirill A. Shutemov4a6c1292012-12-12 13:50:47 -0800211
Matthew Wilcoxfc437042015-09-08 14:58:51 -0700212struct page *get_huge_zero_page(void)
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800213{
214 struct page *zero_page;
215retry:
216 if (likely(atomic_inc_not_zero(&huge_zero_refcount)))
Jason Low4db0c3c2015-04-15 16:14:08 -0700217 return READ_ONCE(huge_zero_page);
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800218
219 zero_page = alloc_pages((GFP_TRANSHUGE | __GFP_ZERO) & ~__GFP_MOVABLE,
220 HPAGE_PMD_ORDER);
Kirill A. Shutemovd8a8e1f2012-12-12 13:51:09 -0800221 if (!zero_page) {
222 count_vm_event(THP_ZERO_PAGE_ALLOC_FAILED);
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700223 return NULL;
Kirill A. Shutemovd8a8e1f2012-12-12 13:51:09 -0800224 }
225 count_vm_event(THP_ZERO_PAGE_ALLOC);
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800226 preempt_disable();
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700227 if (cmpxchg(&huge_zero_page, NULL, zero_page)) {
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800228 preempt_enable();
Yu Zhao5ddacbe2014-10-29 14:50:26 -0700229 __free_pages(zero_page, compound_order(zero_page));
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800230 goto retry;
231 }
232
233 /* We take additional reference here. It will be put back by shrinker */
234 atomic_set(&huge_zero_refcount, 2);
235 preempt_enable();
Jason Low4db0c3c2015-04-15 16:14:08 -0700236 return READ_ONCE(huge_zero_page);
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800237}
238
Kirill A. Shutemovaa88b682016-04-28 16:18:27 -0700239void put_huge_zero_page(void)
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800240{
241 /*
242 * Counter should never go to zero here. Only shrinker can put
243 * last reference.
244 */
245 BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
246}
247
Glauber Costa48896462013-08-28 10:18:15 +1000248static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
249 struct shrink_control *sc)
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800250{
Glauber Costa48896462013-08-28 10:18:15 +1000251 /* we can free zero page only if last reference remains */
252 return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
253}
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800254
Glauber Costa48896462013-08-28 10:18:15 +1000255static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
256 struct shrink_control *sc)
257{
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800258 if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700259 struct page *zero_page = xchg(&huge_zero_page, NULL);
260 BUG_ON(zero_page == NULL);
Yu Zhao5ddacbe2014-10-29 14:50:26 -0700261 __free_pages(zero_page, compound_order(zero_page));
Glauber Costa48896462013-08-28 10:18:15 +1000262 return HPAGE_PMD_NR;
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800263 }
264
265 return 0;
266}
267
268static struct shrinker huge_zero_page_shrinker = {
Glauber Costa48896462013-08-28 10:18:15 +1000269 .count_objects = shrink_huge_zero_page_count,
270 .scan_objects = shrink_huge_zero_page_scan,
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800271 .seeks = DEFAULT_SEEKS,
272};
273
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800274#ifdef CONFIG_SYSFS
Andrea Arcangeliba761492011-01-13 15:46:58 -0800275
Mel Gorman444eb2a42016-03-17 14:19:23 -0700276static ssize_t triple_flag_store(struct kobject *kobj,
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800277 struct kobj_attribute *attr,
278 const char *buf, size_t count,
279 enum transparent_hugepage_flag enabled,
Mel Gorman444eb2a42016-03-17 14:19:23 -0700280 enum transparent_hugepage_flag deferred,
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800281 enum transparent_hugepage_flag req_madv)
282{
Mel Gorman444eb2a42016-03-17 14:19:23 -0700283 if (!memcmp("defer", buf,
284 min(sizeof("defer")-1, count))) {
285 if (enabled == deferred)
286 return -EINVAL;
287 clear_bit(enabled, &transparent_hugepage_flags);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800288 clear_bit(req_madv, &transparent_hugepage_flags);
Mel Gorman444eb2a42016-03-17 14:19:23 -0700289 set_bit(deferred, &transparent_hugepage_flags);
290 } else if (!memcmp("always", buf,
291 min(sizeof("always")-1, count))) {
292 clear_bit(deferred, &transparent_hugepage_flags);
293 clear_bit(req_madv, &transparent_hugepage_flags);
294 set_bit(enabled, &transparent_hugepage_flags);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800295 } else if (!memcmp("madvise", buf,
296 min(sizeof("madvise")-1, count))) {
297 clear_bit(enabled, &transparent_hugepage_flags);
Mel Gorman444eb2a42016-03-17 14:19:23 -0700298 clear_bit(deferred, &transparent_hugepage_flags);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800299 set_bit(req_madv, &transparent_hugepage_flags);
300 } else if (!memcmp("never", buf,
301 min(sizeof("never")-1, count))) {
302 clear_bit(enabled, &transparent_hugepage_flags);
303 clear_bit(req_madv, &transparent_hugepage_flags);
Mel Gorman444eb2a42016-03-17 14:19:23 -0700304 clear_bit(deferred, &transparent_hugepage_flags);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800305 } else
306 return -EINVAL;
307
308 return count;
309}
310
311static ssize_t enabled_show(struct kobject *kobj,
312 struct kobj_attribute *attr, char *buf)
313{
Mel Gorman444eb2a42016-03-17 14:19:23 -0700314 if (test_bit(TRANSPARENT_HUGEPAGE_FLAG, &transparent_hugepage_flags))
315 return sprintf(buf, "[always] madvise never\n");
316 else if (test_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, &transparent_hugepage_flags))
317 return sprintf(buf, "always [madvise] never\n");
318 else
319 return sprintf(buf, "always madvise [never]\n");
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800320}
Mel Gorman444eb2a42016-03-17 14:19:23 -0700321
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800322static ssize_t enabled_store(struct kobject *kobj,
323 struct kobj_attribute *attr,
324 const char *buf, size_t count)
325{
Andrea Arcangeliba761492011-01-13 15:46:58 -0800326 ssize_t ret;
327
Mel Gorman444eb2a42016-03-17 14:19:23 -0700328 ret = triple_flag_store(kobj, attr, buf, count,
329 TRANSPARENT_HUGEPAGE_FLAG,
Andrea Arcangeliba761492011-01-13 15:46:58 -0800330 TRANSPARENT_HUGEPAGE_FLAG,
331 TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG);
332
333 if (ret > 0) {
Xiao Guangrong911891a2012-10-08 16:29:41 -0700334 int err;
335
336 mutex_lock(&khugepaged_mutex);
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700337 err = start_stop_khugepaged();
Xiao Guangrong911891a2012-10-08 16:29:41 -0700338 mutex_unlock(&khugepaged_mutex);
339
Andrea Arcangeliba761492011-01-13 15:46:58 -0800340 if (err)
341 ret = err;
342 }
343
344 return ret;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800345}
346static struct kobj_attribute enabled_attr =
347 __ATTR(enabled, 0644, enabled_show, enabled_store);
348
349static ssize_t single_flag_show(struct kobject *kobj,
350 struct kobj_attribute *attr, char *buf,
351 enum transparent_hugepage_flag flag)
352{
Ben Hutchingse27e6152011-04-14 15:22:21 -0700353 return sprintf(buf, "%d\n",
354 !!test_bit(flag, &transparent_hugepage_flags));
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800355}
Ben Hutchingse27e6152011-04-14 15:22:21 -0700356
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800357static ssize_t single_flag_store(struct kobject *kobj,
358 struct kobj_attribute *attr,
359 const char *buf, size_t count,
360 enum transparent_hugepage_flag flag)
361{
Ben Hutchingse27e6152011-04-14 15:22:21 -0700362 unsigned long value;
363 int ret;
364
365 ret = kstrtoul(buf, 10, &value);
366 if (ret < 0)
367 return ret;
368 if (value > 1)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800369 return -EINVAL;
370
Ben Hutchingse27e6152011-04-14 15:22:21 -0700371 if (value)
372 set_bit(flag, &transparent_hugepage_flags);
373 else
374 clear_bit(flag, &transparent_hugepage_flags);
375
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800376 return count;
377}
378
379/*
380 * Currently defrag only disables __GFP_NOWAIT for allocation. A blind
381 * __GFP_REPEAT is too aggressive, it's never worth swapping tons of
382 * memory just to allocate one more hugepage.
383 */
384static ssize_t defrag_show(struct kobject *kobj,
385 struct kobj_attribute *attr, char *buf)
386{
Mel Gorman444eb2a42016-03-17 14:19:23 -0700387 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
388 return sprintf(buf, "[always] defer madvise never\n");
389 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
390 return sprintf(buf, "always [defer] madvise never\n");
391 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
392 return sprintf(buf, "always defer [madvise] never\n");
393 else
394 return sprintf(buf, "always defer madvise [never]\n");
395
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800396}
397static ssize_t defrag_store(struct kobject *kobj,
398 struct kobj_attribute *attr,
399 const char *buf, size_t count)
400{
Mel Gorman444eb2a42016-03-17 14:19:23 -0700401 return triple_flag_store(kobj, attr, buf, count,
402 TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG,
403 TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG,
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800404 TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG);
405}
406static struct kobj_attribute defrag_attr =
407 __ATTR(defrag, 0644, defrag_show, defrag_store);
408
Kirill A. Shutemov79da5402012-12-12 13:51:12 -0800409static ssize_t use_zero_page_show(struct kobject *kobj,
410 struct kobj_attribute *attr, char *buf)
411{
412 return single_flag_show(kobj, attr, buf,
413 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
414}
415static ssize_t use_zero_page_store(struct kobject *kobj,
416 struct kobj_attribute *attr, const char *buf, size_t count)
417{
418 return single_flag_store(kobj, attr, buf, count,
419 TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG);
420}
421static struct kobj_attribute use_zero_page_attr =
422 __ATTR(use_zero_page, 0644, use_zero_page_show, use_zero_page_store);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800423#ifdef CONFIG_DEBUG_VM
424static ssize_t debug_cow_show(struct kobject *kobj,
425 struct kobj_attribute *attr, char *buf)
426{
427 return single_flag_show(kobj, attr, buf,
428 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
429}
430static ssize_t debug_cow_store(struct kobject *kobj,
431 struct kobj_attribute *attr,
432 const char *buf, size_t count)
433{
434 return single_flag_store(kobj, attr, buf, count,
435 TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG);
436}
437static struct kobj_attribute debug_cow_attr =
438 __ATTR(debug_cow, 0644, debug_cow_show, debug_cow_store);
439#endif /* CONFIG_DEBUG_VM */
440
441static struct attribute *hugepage_attr[] = {
442 &enabled_attr.attr,
443 &defrag_attr.attr,
Kirill A. Shutemov79da5402012-12-12 13:51:12 -0800444 &use_zero_page_attr.attr,
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800445#ifdef CONFIG_DEBUG_VM
446 &debug_cow_attr.attr,
447#endif
448 NULL,
449};
450
451static struct attribute_group hugepage_attr_group = {
452 .attrs = hugepage_attr,
Andrea Arcangeliba761492011-01-13 15:46:58 -0800453};
454
455static ssize_t scan_sleep_millisecs_show(struct kobject *kobj,
456 struct kobj_attribute *attr,
457 char *buf)
458{
459 return sprintf(buf, "%u\n", khugepaged_scan_sleep_millisecs);
460}
461
462static ssize_t scan_sleep_millisecs_store(struct kobject *kobj,
463 struct kobj_attribute *attr,
464 const char *buf, size_t count)
465{
466 unsigned long msecs;
467 int err;
468
Jingoo Han3dbb95f2013-09-11 14:20:25 -0700469 err = kstrtoul(buf, 10, &msecs);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800470 if (err || msecs > UINT_MAX)
471 return -EINVAL;
472
473 khugepaged_scan_sleep_millisecs = msecs;
David Rientjesf0508972016-05-20 16:58:56 -0700474 khugepaged_sleep_expire = 0;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800475 wake_up_interruptible(&khugepaged_wait);
476
477 return count;
478}
479static struct kobj_attribute scan_sleep_millisecs_attr =
480 __ATTR(scan_sleep_millisecs, 0644, scan_sleep_millisecs_show,
481 scan_sleep_millisecs_store);
482
483static ssize_t alloc_sleep_millisecs_show(struct kobject *kobj,
484 struct kobj_attribute *attr,
485 char *buf)
486{
487 return sprintf(buf, "%u\n", khugepaged_alloc_sleep_millisecs);
488}
489
490static ssize_t alloc_sleep_millisecs_store(struct kobject *kobj,
491 struct kobj_attribute *attr,
492 const char *buf, size_t count)
493{
494 unsigned long msecs;
495 int err;
496
Jingoo Han3dbb95f2013-09-11 14:20:25 -0700497 err = kstrtoul(buf, 10, &msecs);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800498 if (err || msecs > UINT_MAX)
499 return -EINVAL;
500
501 khugepaged_alloc_sleep_millisecs = msecs;
David Rientjesf0508972016-05-20 16:58:56 -0700502 khugepaged_sleep_expire = 0;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800503 wake_up_interruptible(&khugepaged_wait);
504
505 return count;
506}
507static struct kobj_attribute alloc_sleep_millisecs_attr =
508 __ATTR(alloc_sleep_millisecs, 0644, alloc_sleep_millisecs_show,
509 alloc_sleep_millisecs_store);
510
511static ssize_t pages_to_scan_show(struct kobject *kobj,
512 struct kobj_attribute *attr,
513 char *buf)
514{
515 return sprintf(buf, "%u\n", khugepaged_pages_to_scan);
516}
517static ssize_t pages_to_scan_store(struct kobject *kobj,
518 struct kobj_attribute *attr,
519 const char *buf, size_t count)
520{
521 int err;
522 unsigned long pages;
523
Jingoo Han3dbb95f2013-09-11 14:20:25 -0700524 err = kstrtoul(buf, 10, &pages);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800525 if (err || !pages || pages > UINT_MAX)
526 return -EINVAL;
527
528 khugepaged_pages_to_scan = pages;
529
530 return count;
531}
532static struct kobj_attribute pages_to_scan_attr =
533 __ATTR(pages_to_scan, 0644, pages_to_scan_show,
534 pages_to_scan_store);
535
536static ssize_t pages_collapsed_show(struct kobject *kobj,
537 struct kobj_attribute *attr,
538 char *buf)
539{
540 return sprintf(buf, "%u\n", khugepaged_pages_collapsed);
541}
542static struct kobj_attribute pages_collapsed_attr =
543 __ATTR_RO(pages_collapsed);
544
545static ssize_t full_scans_show(struct kobject *kobj,
546 struct kobj_attribute *attr,
547 char *buf)
548{
549 return sprintf(buf, "%u\n", khugepaged_full_scans);
550}
551static struct kobj_attribute full_scans_attr =
552 __ATTR_RO(full_scans);
553
554static ssize_t khugepaged_defrag_show(struct kobject *kobj,
555 struct kobj_attribute *attr, char *buf)
556{
557 return single_flag_show(kobj, attr, buf,
558 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
559}
560static ssize_t khugepaged_defrag_store(struct kobject *kobj,
561 struct kobj_attribute *attr,
562 const char *buf, size_t count)
563{
564 return single_flag_store(kobj, attr, buf, count,
565 TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG);
566}
567static struct kobj_attribute khugepaged_defrag_attr =
568 __ATTR(defrag, 0644, khugepaged_defrag_show,
569 khugepaged_defrag_store);
570
571/*
572 * max_ptes_none controls if khugepaged should collapse hugepages over
573 * any unmapped ptes in turn potentially increasing the memory
574 * footprint of the vmas. When max_ptes_none is 0 khugepaged will not
575 * reduce the available free memory in the system as it
576 * runs. Increasing max_ptes_none will instead potentially reduce the
577 * free memory in the system during the khugepaged scan.
578 */
579static ssize_t khugepaged_max_ptes_none_show(struct kobject *kobj,
580 struct kobj_attribute *attr,
581 char *buf)
582{
583 return sprintf(buf, "%u\n", khugepaged_max_ptes_none);
584}
585static ssize_t khugepaged_max_ptes_none_store(struct kobject *kobj,
586 struct kobj_attribute *attr,
587 const char *buf, size_t count)
588{
589 int err;
590 unsigned long max_ptes_none;
591
Jingoo Han3dbb95f2013-09-11 14:20:25 -0700592 err = kstrtoul(buf, 10, &max_ptes_none);
Andrea Arcangeliba761492011-01-13 15:46:58 -0800593 if (err || max_ptes_none > HPAGE_PMD_NR-1)
594 return -EINVAL;
595
596 khugepaged_max_ptes_none = max_ptes_none;
597
598 return count;
599}
600static struct kobj_attribute khugepaged_max_ptes_none_attr =
601 __ATTR(max_ptes_none, 0644, khugepaged_max_ptes_none_show,
602 khugepaged_max_ptes_none_store);
603
Ebru Akagunduz70652f62016-07-26 15:24:59 -0700604static ssize_t khugepaged_max_ptes_swap_show(struct kobject *kobj,
605 struct kobj_attribute *attr,
606 char *buf)
607{
608 return sprintf(buf, "%u\n", khugepaged_max_ptes_swap);
609}
610
611static ssize_t khugepaged_max_ptes_swap_store(struct kobject *kobj,
612 struct kobj_attribute *attr,
613 const char *buf, size_t count)
614{
615 int err;
616 unsigned long max_ptes_swap;
617
618 err = kstrtoul(buf, 10, &max_ptes_swap);
619 if (err || max_ptes_swap > HPAGE_PMD_NR-1)
620 return -EINVAL;
621
622 khugepaged_max_ptes_swap = max_ptes_swap;
623
624 return count;
625}
626
627static struct kobj_attribute khugepaged_max_ptes_swap_attr =
628 __ATTR(max_ptes_swap, 0644, khugepaged_max_ptes_swap_show,
629 khugepaged_max_ptes_swap_store);
630
Andrea Arcangeliba761492011-01-13 15:46:58 -0800631static struct attribute *khugepaged_attr[] = {
632 &khugepaged_defrag_attr.attr,
633 &khugepaged_max_ptes_none_attr.attr,
634 &pages_to_scan_attr.attr,
635 &pages_collapsed_attr.attr,
636 &full_scans_attr.attr,
637 &scan_sleep_millisecs_attr.attr,
638 &alloc_sleep_millisecs_attr.attr,
Ebru Akagunduz70652f62016-07-26 15:24:59 -0700639 &khugepaged_max_ptes_swap_attr.attr,
Andrea Arcangeliba761492011-01-13 15:46:58 -0800640 NULL,
641};
642
643static struct attribute_group khugepaged_attr_group = {
644 .attrs = khugepaged_attr,
645 .name = "khugepaged",
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800646};
Shaohua Li569e5592012-01-12 17:19:11 -0800647
648static int __init hugepage_init_sysfs(struct kobject **hugepage_kobj)
649{
650 int err;
651
652 *hugepage_kobj = kobject_create_and_add("transparent_hugepage", mm_kobj);
653 if (unlikely(!*hugepage_kobj)) {
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700654 pr_err("failed to create transparent hugepage kobject\n");
Shaohua Li569e5592012-01-12 17:19:11 -0800655 return -ENOMEM;
656 }
657
658 err = sysfs_create_group(*hugepage_kobj, &hugepage_attr_group);
659 if (err) {
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700660 pr_err("failed to register transparent hugepage group\n");
Shaohua Li569e5592012-01-12 17:19:11 -0800661 goto delete_obj;
662 }
663
664 err = sysfs_create_group(*hugepage_kobj, &khugepaged_attr_group);
665 if (err) {
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700666 pr_err("failed to register transparent hugepage group\n");
Shaohua Li569e5592012-01-12 17:19:11 -0800667 goto remove_hp_group;
668 }
669
670 return 0;
671
672remove_hp_group:
673 sysfs_remove_group(*hugepage_kobj, &hugepage_attr_group);
674delete_obj:
675 kobject_put(*hugepage_kobj);
676 return err;
677}
678
679static void __init hugepage_exit_sysfs(struct kobject *hugepage_kobj)
680{
681 sysfs_remove_group(hugepage_kobj, &khugepaged_attr_group);
682 sysfs_remove_group(hugepage_kobj, &hugepage_attr_group);
683 kobject_put(hugepage_kobj);
684}
685#else
686static inline int hugepage_init_sysfs(struct kobject **hugepage_kobj)
687{
688 return 0;
689}
690
691static inline void hugepage_exit_sysfs(struct kobject *hugepage_kobj)
692{
693}
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800694#endif /* CONFIG_SYSFS */
695
696static int __init hugepage_init(void)
697{
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800698 int err;
Shaohua Li569e5592012-01-12 17:19:11 -0800699 struct kobject *hugepage_kobj;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800700
Andrea Arcangeli4b7167b2011-01-13 15:47:09 -0800701 if (!has_transparent_hugepage()) {
702 transparent_hugepage_flags = 0;
Shaohua Li569e5592012-01-12 17:19:11 -0800703 return -EINVAL;
Andrea Arcangeli4b7167b2011-01-13 15:47:09 -0800704 }
705
Kirill A. Shutemovff20c2e2016-03-01 09:45:14 +0530706 khugepaged_pages_to_scan = HPAGE_PMD_NR * 8;
707 khugepaged_max_ptes_none = HPAGE_PMD_NR - 1;
Ebru Akagunduz70652f62016-07-26 15:24:59 -0700708 khugepaged_max_ptes_swap = HPAGE_PMD_NR / 8;
Kirill A. Shutemovff20c2e2016-03-01 09:45:14 +0530709 /*
710 * hugepages can't be allocated by the buddy allocator
711 */
712 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER >= MAX_ORDER);
713 /*
714 * we use page->mapping and page->index in second tail page
715 * as list_head: assuming THP order >= 2
716 */
717 MAYBE_BUILD_BUG_ON(HPAGE_PMD_ORDER < 2);
718
Shaohua Li569e5592012-01-12 17:19:11 -0800719 err = hugepage_init_sysfs(&hugepage_kobj);
720 if (err)
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700721 goto err_sysfs;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800722
723 err = khugepaged_slab_init();
724 if (err)
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700725 goto err_slab;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800726
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700727 err = register_shrinker(&huge_zero_page_shrinker);
728 if (err)
729 goto err_hzp_shrinker;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800730 err = register_shrinker(&deferred_split_shrinker);
731 if (err)
732 goto err_split_shrinker;
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800733
Rik van Riel97562cd2011-01-13 15:47:12 -0800734 /*
735 * By default disable transparent hugepages on smaller systems,
736 * where the extra memory used could hurt more than TLB overhead
737 * is likely to save. The admin can still enable it through /sys.
738 */
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700739 if (totalram_pages < (512 << (20 - PAGE_SHIFT))) {
Rik van Riel97562cd2011-01-13 15:47:12 -0800740 transparent_hugepage_flags = 0;
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700741 return 0;
742 }
Rik van Riel97562cd2011-01-13 15:47:12 -0800743
Kirill A. Shutemov79553da2015-04-15 16:14:56 -0700744 err = start_stop_khugepaged();
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700745 if (err)
746 goto err_khugepaged;
Andrea Arcangeliba761492011-01-13 15:46:58 -0800747
Shaohua Li569e5592012-01-12 17:19:11 -0800748 return 0;
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700749err_khugepaged:
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800750 unregister_shrinker(&deferred_split_shrinker);
751err_split_shrinker:
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700752 unregister_shrinker(&huge_zero_page_shrinker);
753err_hzp_shrinker:
754 khugepaged_slab_exit();
755err_slab:
Shaohua Li569e5592012-01-12 17:19:11 -0800756 hugepage_exit_sysfs(hugepage_kobj);
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -0700757err_sysfs:
Andrea Arcangeliba761492011-01-13 15:46:58 -0800758 return err;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800759}
Paul Gortmakera64fb3c2014-01-23 15:53:30 -0800760subsys_initcall(hugepage_init);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800761
762static int __init setup_transparent_hugepage(char *str)
763{
764 int ret = 0;
765 if (!str)
766 goto out;
767 if (!strcmp(str, "always")) {
768 set_bit(TRANSPARENT_HUGEPAGE_FLAG,
769 &transparent_hugepage_flags);
770 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
771 &transparent_hugepage_flags);
772 ret = 1;
773 } else if (!strcmp(str, "madvise")) {
774 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
775 &transparent_hugepage_flags);
776 set_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
777 &transparent_hugepage_flags);
778 ret = 1;
779 } else if (!strcmp(str, "never")) {
780 clear_bit(TRANSPARENT_HUGEPAGE_FLAG,
781 &transparent_hugepage_flags);
782 clear_bit(TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG,
783 &transparent_hugepage_flags);
784 ret = 1;
785 }
786out:
787 if (!ret)
Andrew Mortonae3a8c12014-06-04 16:06:58 -0700788 pr_warn("transparent_hugepage= cannot parse, ignored\n");
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800789 return ret;
790}
791__setup("transparent_hugepage=", setup_transparent_hugepage);
792
Mel Gormanb32967f2012-11-19 12:35:47 +0000793pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800794{
795 if (likely(vma->vm_flags & VM_WRITE))
796 pmd = pmd_mkwrite(pmd);
797 return pmd;
798}
799
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800800static inline struct list_head *page_deferred_list(struct page *page)
801{
802 /*
803 * ->lru in the tail pages is occupied by compound_head.
804 * Let's use ->mapping + ->index in the second tail page as list_head.
805 */
806 return (struct list_head *)&page[2].mapping;
807}
808
809void prep_transhuge_page(struct page *page)
810{
811 /*
812 * we use page->mapping and page->indexlru in second tail page
813 * as list_head: assuming THP order >= 2
814 */
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800815
816 INIT_LIST_HEAD(page_deferred_list(page));
817 set_compound_page_dtor(page, TRANSHUGE_PAGE_DTOR);
818}
819
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700820static int __do_huge_pmd_anonymous_page(struct fault_env *fe, struct page *page,
821 gfp_t gfp)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800822{
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700823 struct vm_area_struct *vma = fe->vma;
Johannes Weiner00501b52014-08-08 14:19:20 -0700824 struct mem_cgroup *memcg;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800825 pgtable_t pgtable;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700826 unsigned long haddr = fe->address & HPAGE_PMD_MASK;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800827
Sasha Levin309381fea2014-01-23 15:52:54 -0800828 VM_BUG_ON_PAGE(!PageCompound(page), page);
Johannes Weiner00501b52014-08-08 14:19:20 -0700829
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700830 if (mem_cgroup_try_charge(page, vma->vm_mm, gfp, &memcg, true)) {
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700831 put_page(page);
832 count_vm_event(THP_FAULT_FALLBACK);
833 return VM_FAULT_FALLBACK;
834 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800835
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700836 pgtable = pte_alloc_one(vma->vm_mm, haddr);
Johannes Weiner00501b52014-08-08 14:19:20 -0700837 if (unlikely(!pgtable)) {
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800838 mem_cgroup_cancel_charge(page, memcg, true);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700839 put_page(page);
Johannes Weiner00501b52014-08-08 14:19:20 -0700840 return VM_FAULT_OOM;
841 }
842
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800843 clear_huge_page(page, haddr, HPAGE_PMD_NR);
Minchan Kim52f37622013-04-29 15:08:15 -0700844 /*
845 * The memory barrier inside __SetPageUptodate makes sure that
846 * clear_huge_page writes become visible before the set_pmd_at()
847 * write.
848 */
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800849 __SetPageUptodate(page);
850
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700851 fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
852 if (unlikely(!pmd_none(*fe->pmd))) {
853 spin_unlock(fe->ptl);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800854 mem_cgroup_cancel_charge(page, memcg, true);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800855 put_page(page);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700856 pte_free(vma->vm_mm, pgtable);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800857 } else {
858 pmd_t entry;
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700859
860 /* Deliver the page fault to userland */
861 if (userfaultfd_missing(vma)) {
862 int ret;
863
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700864 spin_unlock(fe->ptl);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800865 mem_cgroup_cancel_charge(page, memcg, true);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700866 put_page(page);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700867 pte_free(vma->vm_mm, pgtable);
868 ret = handle_userfault(fe, VM_UFFD_MISSING);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700869 VM_BUG_ON(ret & VM_FAULT_FALLBACK);
870 return ret;
871 }
872
Kirill A. Shutemov31223592013-09-12 15:14:01 -0700873 entry = mk_huge_pmd(page, vma->vm_page_prot);
874 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -0800875 page_add_new_anon_rmap(page, vma, haddr, true);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -0800876 mem_cgroup_commit_charge(page, memcg, false, true);
Johannes Weiner00501b52014-08-08 14:19:20 -0700877 lru_cache_add_active_or_unevictable(page, vma);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700878 pgtable_trans_huge_deposit(vma->vm_mm, fe->pmd, pgtable);
879 set_pmd_at(vma->vm_mm, haddr, fe->pmd, entry);
880 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
881 atomic_long_inc(&vma->vm_mm->nr_ptes);
882 spin_unlock(fe->ptl);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700883 count_vm_event(THP_FAULT_ALLOC);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800884 }
885
David Rientjesaa2e8782012-05-29 15:06:17 -0700886 return 0;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800887}
888
Mel Gorman444eb2a42016-03-17 14:19:23 -0700889/*
890 * If THP is set to always then directly reclaim/compact as necessary
891 * If set to defer then do no reclaim and defer to khugepaged
892 * If set to madvise and the VMA is flagged then directly reclaim/compact
893 */
894static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma)
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -0800895{
Mel Gorman444eb2a42016-03-17 14:19:23 -0700896 gfp_t reclaim_flags = 0;
897
898 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags) &&
899 (vma->vm_flags & VM_HUGEPAGE))
900 reclaim_flags = __GFP_DIRECT_RECLAIM;
901 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
902 reclaim_flags = __GFP_KSWAPD_RECLAIM;
903 else if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
904 reclaim_flags = __GFP_DIRECT_RECLAIM;
905
906 return GFP_TRANSHUGE | reclaim_flags;
907}
908
909/* Defrag for khugepaged will enter direct reclaim/compaction if necessary */
910static inline gfp_t alloc_hugepage_khugepaged_gfpmask(void)
911{
912 return GFP_TRANSHUGE | (khugepaged_defrag() ? __GFP_DIRECT_RECLAIM : 0);
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -0800913}
914
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -0800915/* Caller must hold page table lock. */
Kirill A. Shutemovd295e342015-09-08 14:59:34 -0700916static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -0800917 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700918 struct page *zero_page)
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -0800919{
920 pmd_t entry;
Andrew Morton7c414162015-09-08 14:58:43 -0700921 if (!pmd_none(*pmd))
922 return false;
Kirill A. Shutemov5918d102013-04-29 15:08:44 -0700923 entry = mk_pmd(zero_page, vma->vm_page_prot);
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -0800924 entry = pmd_mkhuge(entry);
Matthew Wilcox12c9d702016-02-02 16:57:57 -0800925 if (pgtable)
926 pgtable_trans_huge_deposit(mm, pmd, pgtable);
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -0800927 set_pmd_at(mm, haddr, pmd, entry);
Kirill A. Shutemove1f56c82013-11-14 14:30:48 -0800928 atomic_long_inc(&mm->nr_ptes);
Andrew Morton7c414162015-09-08 14:58:43 -0700929 return true;
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -0800930}
931
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700932int do_huge_pmd_anonymous_page(struct fault_env *fe)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800933{
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700934 struct vm_area_struct *vma = fe->vma;
Aneesh Kumar K.V077fcf12015-02-11 15:27:12 -0800935 gfp_t gfp;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800936 struct page *page;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700937 unsigned long haddr = fe->address & HPAGE_PMD_MASK;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800938
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700939 if (haddr < vma->vm_start || haddr + HPAGE_PMD_SIZE > vma->vm_end)
Kirill A. Shutemovc0292552013-09-12 15:14:05 -0700940 return VM_FAULT_FALLBACK;
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700941 if (unlikely(anon_vma_prepare(vma)))
942 return VM_FAULT_OOM;
David Rientjes6d50e602014-10-29 14:50:31 -0700943 if (unlikely(khugepaged_enter(vma, vma->vm_flags)))
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700944 return VM_FAULT_OOM;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700945 if (!(fe->flags & FAULT_FLAG_WRITE) &&
946 !mm_forbids_zeropage(vma->vm_mm) &&
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700947 transparent_hugepage_use_zero_page()) {
948 pgtable_t pgtable;
949 struct page *zero_page;
950 bool set;
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700951 int ret;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700952 pgtable = pte_alloc_one(vma->vm_mm, haddr);
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700953 if (unlikely(!pgtable))
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800954 return VM_FAULT_OOM;
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700955 zero_page = get_huge_zero_page();
956 if (unlikely(!zero_page)) {
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700957 pte_free(vma->vm_mm, pgtable);
Andi Kleen81ab4202011-04-14 15:22:06 -0700958 count_vm_event(THP_FAULT_FALLBACK);
Kirill A. Shutemovc0292552013-09-12 15:14:05 -0700959 return VM_FAULT_FALLBACK;
Andi Kleen81ab4202011-04-14 15:22:06 -0700960 }
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700961 fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700962 ret = 0;
963 set = false;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700964 if (pmd_none(*fe->pmd)) {
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700965 if (userfaultfd_missing(vma)) {
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700966 spin_unlock(fe->ptl);
967 ret = handle_userfault(fe, VM_UFFD_MISSING);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700968 VM_BUG_ON(ret & VM_FAULT_FALLBACK);
969 } else {
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700970 set_huge_zero_page(pgtable, vma->vm_mm, vma,
971 haddr, fe->pmd, zero_page);
972 spin_unlock(fe->ptl);
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700973 set = true;
974 }
975 } else
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700976 spin_unlock(fe->ptl);
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700977 if (!set) {
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700978 pte_free(vma->vm_mm, pgtable);
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700979 put_huge_zero_page();
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -0800980 }
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -0700981 return ret;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800982 }
Mel Gorman444eb2a42016-03-17 14:19:23 -0700983 gfp = alloc_hugepage_direct_gfpmask(vma);
Aneesh Kumar K.V077fcf12015-02-11 15:27:12 -0800984 page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700985 if (unlikely(!page)) {
986 count_vm_event(THP_FAULT_FALLBACK);
Kirill A. Shutemovc0292552013-09-12 15:14:05 -0700987 return VM_FAULT_FALLBACK;
Kirill A. Shutemov128ec032013-09-12 15:14:03 -0700988 }
Kirill A. Shutemov9a982252016-01-15 16:54:17 -0800989 prep_transhuge_page(page);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -0700990 return __do_huge_pmd_anonymous_page(fe, page, gfp);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800991}
992
Matthew Wilcoxae18d6d2015-09-08 14:59:14 -0700993static void insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
Dan Williamsf25748e32016-01-15 16:56:43 -0800994 pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write)
Matthew Wilcox5cad4652015-09-08 14:58:54 -0700995{
996 struct mm_struct *mm = vma->vm_mm;
997 pmd_t entry;
998 spinlock_t *ptl;
999
1000 ptl = pmd_lock(mm, pmd);
Dan Williamsf25748e32016-01-15 16:56:43 -08001001 entry = pmd_mkhuge(pfn_t_pmd(pfn, prot));
1002 if (pfn_t_devmap(pfn))
1003 entry = pmd_mkdevmap(entry);
Ross Zwisler01871e52016-01-15 16:56:02 -08001004 if (write) {
1005 entry = pmd_mkyoung(pmd_mkdirty(entry));
1006 entry = maybe_pmd_mkwrite(entry, vma);
Matthew Wilcox5cad4652015-09-08 14:58:54 -07001007 }
Ross Zwisler01871e52016-01-15 16:56:02 -08001008 set_pmd_at(mm, addr, pmd, entry);
1009 update_mmu_cache_pmd(vma, addr, pmd);
Matthew Wilcox5cad4652015-09-08 14:58:54 -07001010 spin_unlock(ptl);
Matthew Wilcox5cad4652015-09-08 14:58:54 -07001011}
1012
1013int vmf_insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr,
Dan Williamsf25748e32016-01-15 16:56:43 -08001014 pmd_t *pmd, pfn_t pfn, bool write)
Matthew Wilcox5cad4652015-09-08 14:58:54 -07001015{
1016 pgprot_t pgprot = vma->vm_page_prot;
1017 /*
1018 * If we had pmd_special, we could avoid all these restrictions,
1019 * but we need to be consistent with PTEs and architectures that
1020 * can't support a 'special' bit.
1021 */
1022 BUG_ON(!(vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)));
1023 BUG_ON((vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) ==
1024 (VM_PFNMAP|VM_MIXEDMAP));
1025 BUG_ON((vma->vm_flags & VM_PFNMAP) && is_cow_mapping(vma->vm_flags));
Dan Williamsf25748e32016-01-15 16:56:43 -08001026 BUG_ON(!pfn_t_devmap(pfn));
Matthew Wilcox5cad4652015-09-08 14:58:54 -07001027
1028 if (addr < vma->vm_start || addr >= vma->vm_end)
1029 return VM_FAULT_SIGBUS;
1030 if (track_pfn_insert(vma, &pgprot, pfn))
1031 return VM_FAULT_SIGBUS;
Matthew Wilcoxae18d6d2015-09-08 14:59:14 -07001032 insert_pfn_pmd(vma, addr, pmd, pfn, pgprot, write);
1033 return VM_FAULT_NOPAGE;
Matthew Wilcox5cad4652015-09-08 14:58:54 -07001034}
Dan Williamsdee41072016-05-14 12:20:44 -07001035EXPORT_SYMBOL_GPL(vmf_insert_pfn_pmd);
Matthew Wilcox5cad4652015-09-08 14:58:54 -07001036
Dan Williams3565fce2016-01-15 16:56:55 -08001037static void touch_pmd(struct vm_area_struct *vma, unsigned long addr,
1038 pmd_t *pmd)
1039{
1040 pmd_t _pmd;
1041
1042 /*
1043 * We should set the dirty bit only for FOLL_WRITE but for now
1044 * the dirty bit in the pmd is meaningless. And if the dirty
1045 * bit will become meaningful and we'll only set it with
1046 * FOLL_WRITE, an atomic set_bit will be required on the pmd to
1047 * set the young bit, instead of the current set_pmd_at.
1048 */
1049 _pmd = pmd_mkyoung(pmd_mkdirty(*pmd));
1050 if (pmdp_set_access_flags(vma, addr & HPAGE_PMD_MASK,
1051 pmd, _pmd, 1))
1052 update_mmu_cache_pmd(vma, addr, pmd);
1053}
1054
1055struct page *follow_devmap_pmd(struct vm_area_struct *vma, unsigned long addr,
1056 pmd_t *pmd, int flags)
1057{
1058 unsigned long pfn = pmd_pfn(*pmd);
1059 struct mm_struct *mm = vma->vm_mm;
1060 struct dev_pagemap *pgmap;
1061 struct page *page;
1062
1063 assert_spin_locked(pmd_lockptr(mm, pmd));
1064
1065 if (flags & FOLL_WRITE && !pmd_write(*pmd))
1066 return NULL;
1067
1068 if (pmd_present(*pmd) && pmd_devmap(*pmd))
1069 /* pass */;
1070 else
1071 return NULL;
1072
1073 if (flags & FOLL_TOUCH)
1074 touch_pmd(vma, addr, pmd);
1075
1076 /*
1077 * device mapped pages can only be returned if the
1078 * caller will manage the page reference count.
1079 */
1080 if (!(flags & FOLL_GET))
1081 return ERR_PTR(-EEXIST);
1082
1083 pfn += (addr & ~PMD_MASK) >> PAGE_SHIFT;
1084 pgmap = get_dev_pagemap(pfn, NULL);
1085 if (!pgmap)
1086 return ERR_PTR(-EFAULT);
1087 page = pfn_to_page(pfn);
1088 get_page(page);
1089 put_dev_pagemap(pgmap);
1090
1091 return page;
1092}
1093
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001094int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm,
1095 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
1096 struct vm_area_struct *vma)
1097{
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001098 spinlock_t *dst_ptl, *src_ptl;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001099 struct page *src_page;
1100 pmd_t pmd;
Matthew Wilcox12c9d702016-02-02 16:57:57 -08001101 pgtable_t pgtable = NULL;
Kirill A. Shutemov628d47c2016-07-26 15:25:42 -07001102 int ret = -ENOMEM;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001103
Kirill A. Shutemov628d47c2016-07-26 15:25:42 -07001104 /* Skip if can be re-fill on fault */
1105 if (!vma_is_anonymous(vma))
1106 return 0;
1107
1108 pgtable = pte_alloc_one(dst_mm, addr);
1109 if (unlikely(!pgtable))
1110 goto out;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001111
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001112 dst_ptl = pmd_lock(dst_mm, dst_pmd);
1113 src_ptl = pmd_lockptr(src_mm, src_pmd);
1114 spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001115
1116 ret = -EAGAIN;
1117 pmd = *src_pmd;
Kirill A. Shutemov628d47c2016-07-26 15:25:42 -07001118 if (unlikely(!pmd_trans_huge(pmd))) {
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001119 pte_free(dst_mm, pgtable);
1120 goto out_unlock;
1121 }
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -08001122 /*
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001123 * When page table lock is held, the huge zero pmd should not be
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -08001124 * under splitting since we don't split the page itself, only pmd to
1125 * a page table.
1126 */
1127 if (is_huge_zero_pmd(pmd)) {
Kirill A. Shutemov5918d102013-04-29 15:08:44 -07001128 struct page *zero_page;
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -08001129 /*
1130 * get_huge_zero_page() will never allocate a new page here,
1131 * since we already have a zero page to copy. It just takes a
1132 * reference.
1133 */
Kirill A. Shutemov5918d102013-04-29 15:08:44 -07001134 zero_page = get_huge_zero_page();
Andrea Arcangeli6b251fc2015-09-04 15:46:20 -07001135 set_huge_zero_page(pgtable, dst_mm, vma, addr, dst_pmd,
Kirill A. Shutemov5918d102013-04-29 15:08:44 -07001136 zero_page);
Kirill A. Shutemovfc9fe822012-12-12 13:50:51 -08001137 ret = 0;
1138 goto out_unlock;
1139 }
Mel Gormande466bd2013-12-18 17:08:42 -08001140
Kirill A. Shutemov628d47c2016-07-26 15:25:42 -07001141 src_page = pmd_page(pmd);
1142 VM_BUG_ON_PAGE(!PageHead(src_page), src_page);
1143 get_page(src_page);
1144 page_dup_rmap(src_page, true);
1145 add_mm_counter(dst_mm, MM_ANONPAGES, HPAGE_PMD_NR);
1146 atomic_long_inc(&dst_mm->nr_ptes);
1147 pgtable_trans_huge_deposit(dst_mm, dst_pmd, pgtable);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001148
1149 pmdp_set_wrprotect(src_mm, addr, src_pmd);
1150 pmd = pmd_mkold(pmd_wrprotect(pmd));
1151 set_pmd_at(dst_mm, addr, dst_pmd, pmd);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001152
1153 ret = 0;
1154out_unlock:
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001155 spin_unlock(src_ptl);
1156 spin_unlock(dst_ptl);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001157out:
1158 return ret;
1159}
1160
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001161void huge_pmd_set_accessed(struct fault_env *fe, pmd_t orig_pmd)
Will Deacona1dd4502012-12-11 16:01:27 -08001162{
1163 pmd_t entry;
1164 unsigned long haddr;
1165
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001166 fe->ptl = pmd_lock(fe->vma->vm_mm, fe->pmd);
1167 if (unlikely(!pmd_same(*fe->pmd, orig_pmd)))
Will Deacona1dd4502012-12-11 16:01:27 -08001168 goto unlock;
1169
1170 entry = pmd_mkyoung(orig_pmd);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001171 haddr = fe->address & HPAGE_PMD_MASK;
1172 if (pmdp_set_access_flags(fe->vma, haddr, fe->pmd, entry,
1173 fe->flags & FAULT_FLAG_WRITE))
1174 update_mmu_cache_pmd(fe->vma, fe->address, fe->pmd);
Will Deacona1dd4502012-12-11 16:01:27 -08001175
1176unlock:
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001177 spin_unlock(fe->ptl);
Will Deacona1dd4502012-12-11 16:01:27 -08001178}
1179
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001180static int do_huge_pmd_wp_page_fallback(struct fault_env *fe, pmd_t orig_pmd,
1181 struct page *page)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001182{
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001183 struct vm_area_struct *vma = fe->vma;
1184 unsigned long haddr = fe->address & HPAGE_PMD_MASK;
Johannes Weiner00501b52014-08-08 14:19:20 -07001185 struct mem_cgroup *memcg;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001186 pgtable_t pgtable;
1187 pmd_t _pmd;
1188 int ret = 0, i;
1189 struct page **pages;
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001190 unsigned long mmun_start; /* For mmu_notifiers */
1191 unsigned long mmun_end; /* For mmu_notifiers */
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001192
1193 pages = kmalloc(sizeof(struct page *) * HPAGE_PMD_NR,
1194 GFP_KERNEL);
1195 if (unlikely(!pages)) {
1196 ret |= VM_FAULT_OOM;
1197 goto out;
1198 }
1199
1200 for (i = 0; i < HPAGE_PMD_NR; i++) {
Andi Kleencc5d4622011-03-22 16:33:13 -07001201 pages[i] = alloc_page_vma_node(GFP_HIGHUSER_MOVABLE |
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001202 __GFP_OTHER_NODE, vma,
1203 fe->address, page_to_nid(page));
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001204 if (unlikely(!pages[i] ||
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001205 mem_cgroup_try_charge(pages[i], vma->vm_mm,
1206 GFP_KERNEL, &memcg, false))) {
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001207 if (pages[i])
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001208 put_page(pages[i]);
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001209 while (--i >= 0) {
Johannes Weiner00501b52014-08-08 14:19:20 -07001210 memcg = (void *)page_private(pages[i]);
1211 set_page_private(pages[i], 0);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08001212 mem_cgroup_cancel_charge(pages[i], memcg,
1213 false);
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001214 put_page(pages[i]);
1215 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001216 kfree(pages);
1217 ret |= VM_FAULT_OOM;
1218 goto out;
1219 }
Johannes Weiner00501b52014-08-08 14:19:20 -07001220 set_page_private(pages[i], (unsigned long)memcg);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001221 }
1222
1223 for (i = 0; i < HPAGE_PMD_NR; i++) {
1224 copy_user_highpage(pages[i], page + i,
Hillf Danton0089e482011-10-31 17:09:38 -07001225 haddr + PAGE_SIZE * i, vma);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001226 __SetPageUptodate(pages[i]);
1227 cond_resched();
1228 }
1229
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001230 mmun_start = haddr;
1231 mmun_end = haddr + HPAGE_PMD_SIZE;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001232 mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001233
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001234 fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
1235 if (unlikely(!pmd_same(*fe->pmd, orig_pmd)))
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001236 goto out_free_pages;
Sasha Levin309381fea2014-01-23 15:52:54 -08001237 VM_BUG_ON_PAGE(!PageHead(page), page);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001238
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001239 pmdp_huge_clear_flush_notify(vma, haddr, fe->pmd);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001240 /* leave pmd empty until pte is filled */
1241
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001242 pgtable = pgtable_trans_huge_withdraw(vma->vm_mm, fe->pmd);
1243 pmd_populate(vma->vm_mm, &_pmd, pgtable);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001244
1245 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001246 pte_t entry;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001247 entry = mk_pte(pages[i], vma->vm_page_prot);
1248 entry = maybe_mkwrite(pte_mkdirty(entry), vma);
Johannes Weiner00501b52014-08-08 14:19:20 -07001249 memcg = (void *)page_private(pages[i]);
1250 set_page_private(pages[i], 0);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001251 page_add_new_anon_rmap(pages[i], fe->vma, haddr, false);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08001252 mem_cgroup_commit_charge(pages[i], memcg, false, false);
Johannes Weiner00501b52014-08-08 14:19:20 -07001253 lru_cache_add_active_or_unevictable(pages[i], vma);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001254 fe->pte = pte_offset_map(&_pmd, haddr);
1255 VM_BUG_ON(!pte_none(*fe->pte));
1256 set_pte_at(vma->vm_mm, haddr, fe->pte, entry);
1257 pte_unmap(fe->pte);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001258 }
1259 kfree(pages);
1260
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001261 smp_wmb(); /* make pte visible before pmd */
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001262 pmd_populate(vma->vm_mm, fe->pmd, pgtable);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08001263 page_remove_rmap(page, true);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001264 spin_unlock(fe->ptl);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001265
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001266 mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001267
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001268 ret |= VM_FAULT_WRITE;
1269 put_page(page);
1270
1271out:
1272 return ret;
1273
1274out_free_pages:
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001275 spin_unlock(fe->ptl);
1276 mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001277 for (i = 0; i < HPAGE_PMD_NR; i++) {
Johannes Weiner00501b52014-08-08 14:19:20 -07001278 memcg = (void *)page_private(pages[i]);
1279 set_page_private(pages[i], 0);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08001280 mem_cgroup_cancel_charge(pages[i], memcg, false);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001281 put_page(pages[i]);
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001282 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001283 kfree(pages);
1284 goto out;
1285}
1286
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001287int do_huge_pmd_wp_page(struct fault_env *fe, pmd_t orig_pmd)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001288{
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001289 struct vm_area_struct *vma = fe->vma;
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001290 struct page *page = NULL, *new_page;
Johannes Weiner00501b52014-08-08 14:19:20 -07001291 struct mem_cgroup *memcg;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001292 unsigned long haddr = fe->address & HPAGE_PMD_MASK;
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001293 unsigned long mmun_start; /* For mmu_notifiers */
1294 unsigned long mmun_end; /* For mmu_notifiers */
Michal Hocko3b363692015-04-15 16:13:29 -07001295 gfp_t huge_gfp; /* for allocation and charge */
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001296 int ret = 0;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001297
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001298 fe->ptl = pmd_lockptr(vma->vm_mm, fe->pmd);
Sasha Levin81d1b092014-10-09 15:28:10 -07001299 VM_BUG_ON_VMA(!vma->anon_vma, vma);
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001300 if (is_huge_zero_pmd(orig_pmd))
1301 goto alloc;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001302 spin_lock(fe->ptl);
1303 if (unlikely(!pmd_same(*fe->pmd, orig_pmd)))
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001304 goto out_unlock;
1305
1306 page = pmd_page(orig_pmd);
Sasha Levin309381fea2014-01-23 15:52:54 -08001307 VM_BUG_ON_PAGE(!PageCompound(page) || !PageHead(page), page);
Kirill A. Shutemov1f25fe22016-01-15 16:52:24 -08001308 /*
1309 * We can only reuse the page if nobody else maps the huge page or it's
Andrea Arcangeli6d0a07e2016-05-12 15:42:25 -07001310 * part.
Kirill A. Shutemov1f25fe22016-01-15 16:52:24 -08001311 */
Andrea Arcangeli6d0a07e2016-05-12 15:42:25 -07001312 if (page_trans_huge_mapcount(page, NULL) == 1) {
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001313 pmd_t entry;
1314 entry = pmd_mkyoung(orig_pmd);
1315 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001316 if (pmdp_set_access_flags(vma, haddr, fe->pmd, entry, 1))
1317 update_mmu_cache_pmd(vma, fe->address, fe->pmd);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001318 ret |= VM_FAULT_WRITE;
1319 goto out_unlock;
1320 }
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08001321 get_page(page);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001322 spin_unlock(fe->ptl);
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001323alloc:
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001324 if (transparent_hugepage_enabled(vma) &&
Aneesh Kumar K.V077fcf12015-02-11 15:27:12 -08001325 !transparent_hugepage_debug_cow()) {
Mel Gorman444eb2a42016-03-17 14:19:23 -07001326 huge_gfp = alloc_hugepage_direct_gfpmask(vma);
Michal Hocko3b363692015-04-15 16:13:29 -07001327 new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
Aneesh Kumar K.V077fcf12015-02-11 15:27:12 -08001328 } else
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001329 new_page = NULL;
1330
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08001331 if (likely(new_page)) {
1332 prep_transhuge_page(new_page);
1333 } else {
Hugh Dickinseecc1e42014-01-12 01:25:21 -08001334 if (!page) {
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001335 split_huge_pmd(vma, fe->pmd, fe->address);
Kirill A. Shutemove9b71ca2014-04-03 14:48:17 -07001336 ret |= VM_FAULT_FALLBACK;
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001337 } else {
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001338 ret = do_huge_pmd_wp_page_fallback(fe, orig_pmd, page);
Kirill A. Shutemov9845cbb2014-02-25 15:01:42 -08001339 if (ret & VM_FAULT_OOM) {
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001340 split_huge_pmd(vma, fe->pmd, fe->address);
Kirill A. Shutemov9845cbb2014-02-25 15:01:42 -08001341 ret |= VM_FAULT_FALLBACK;
1342 }
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08001343 put_page(page);
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001344 }
David Rientjes17766dd2013-09-12 15:14:06 -07001345 count_vm_event(THP_FAULT_FALLBACK);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001346 goto out;
1347 }
1348
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001349 if (unlikely(mem_cgroup_try_charge(new_page, vma->vm_mm,
1350 huge_gfp, &memcg, true))) {
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001351 put_page(new_page);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001352 split_huge_pmd(vma, fe->pmd, fe->address);
1353 if (page)
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08001354 put_page(page);
Kirill A. Shutemov9845cbb2014-02-25 15:01:42 -08001355 ret |= VM_FAULT_FALLBACK;
David Rientjes17766dd2013-09-12 15:14:06 -07001356 count_vm_event(THP_FAULT_FALLBACK);
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001357 goto out;
1358 }
1359
David Rientjes17766dd2013-09-12 15:14:06 -07001360 count_vm_event(THP_FAULT_ALLOC);
1361
Hugh Dickinseecc1e42014-01-12 01:25:21 -08001362 if (!page)
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001363 clear_huge_page(new_page, haddr, HPAGE_PMD_NR);
1364 else
1365 copy_user_huge_page(new_page, page, haddr, vma, HPAGE_PMD_NR);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001366 __SetPageUptodate(new_page);
1367
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001368 mmun_start = haddr;
1369 mmun_end = haddr + HPAGE_PMD_SIZE;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001370 mmu_notifier_invalidate_range_start(vma->vm_mm, mmun_start, mmun_end);
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001371
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001372 spin_lock(fe->ptl);
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001373 if (page)
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08001374 put_page(page);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001375 if (unlikely(!pmd_same(*fe->pmd, orig_pmd))) {
1376 spin_unlock(fe->ptl);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08001377 mem_cgroup_cancel_charge(new_page, memcg, true);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001378 put_page(new_page);
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001379 goto out_mn;
Andrea Arcangelib9bbfbe2011-01-13 15:46:57 -08001380 } else {
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001381 pmd_t entry;
Kirill A. Shutemov31223592013-09-12 15:14:01 -07001382 entry = mk_huge_pmd(new_page, vma->vm_page_prot);
1383 entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001384 pmdp_huge_clear_flush_notify(vma, haddr, fe->pmd);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08001385 page_add_new_anon_rmap(new_page, vma, haddr, true);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08001386 mem_cgroup_commit_charge(new_page, memcg, false, true);
Johannes Weiner00501b52014-08-08 14:19:20 -07001387 lru_cache_add_active_or_unevictable(new_page, vma);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001388 set_pmd_at(vma->vm_mm, haddr, fe->pmd, entry);
1389 update_mmu_cache_pmd(vma, fe->address, fe->pmd);
Hugh Dickinseecc1e42014-01-12 01:25:21 -08001390 if (!page) {
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001391 add_mm_counter(vma->vm_mm, MM_ANONPAGES, HPAGE_PMD_NR);
Kirill A. Shutemov97ae1742012-12-12 13:51:06 -08001392 put_huge_zero_page();
1393 } else {
Sasha Levin309381fea2014-01-23 15:52:54 -08001394 VM_BUG_ON_PAGE(!PageHead(page), page);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08001395 page_remove_rmap(page, true);
Kirill A. Shutemov93b47962012-12-12 13:50:54 -08001396 put_page(page);
1397 }
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001398 ret |= VM_FAULT_WRITE;
1399 }
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001400 spin_unlock(fe->ptl);
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001401out_mn:
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001402 mmu_notifier_invalidate_range_end(vma->vm_mm, mmun_start, mmun_end);
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07001403out:
1404 return ret;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001405out_unlock:
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001406 spin_unlock(fe->ptl);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001407 return ret;
1408}
1409
David Rientjesb676b292012-10-08 16:34:03 -07001410struct page *follow_trans_huge_pmd(struct vm_area_struct *vma,
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001411 unsigned long addr,
1412 pmd_t *pmd,
1413 unsigned int flags)
1414{
David Rientjesb676b292012-10-08 16:34:03 -07001415 struct mm_struct *mm = vma->vm_mm;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001416 struct page *page = NULL;
1417
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08001418 assert_spin_locked(pmd_lockptr(mm, pmd));
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001419
1420 if (flags & FOLL_WRITE && !pmd_write(*pmd))
1421 goto out;
1422
Kirill A. Shutemov85facf22013-02-04 14:28:42 -08001423 /* Avoid dumping huge zero page */
1424 if ((flags & FOLL_DUMP) && is_huge_zero_pmd(*pmd))
1425 return ERR_PTR(-EFAULT);
1426
Mel Gorman2b4847e2013-12-18 17:08:32 -08001427 /* Full NUMA hinting faults to serialise migration in fault paths */
Mel Gorman8a0516e2015-02-12 14:58:22 -08001428 if ((flags & FOLL_NUMA) && pmd_protnone(*pmd))
Mel Gorman2b4847e2013-12-18 17:08:32 -08001429 goto out;
1430
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001431 page = pmd_page(*pmd);
Sasha Levin309381fea2014-01-23 15:52:54 -08001432 VM_BUG_ON_PAGE(!PageHead(page), page);
Dan Williams3565fce2016-01-15 16:56:55 -08001433 if (flags & FOLL_TOUCH)
1434 touch_pmd(vma, addr, pmd);
Eric B Munsonde60f5f2015-11-05 18:51:36 -08001435 if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
Kirill A. Shutemove90309c2016-01-15 16:54:33 -08001436 /*
1437 * We don't mlock() pte-mapped THPs. This way we can avoid
1438 * leaking mlocked pages into non-VM_LOCKED VMAs.
1439 *
Kirill A. Shutemov9a73f612016-07-26 15:25:53 -07001440 * For anon THP:
1441 *
Kirill A. Shutemove90309c2016-01-15 16:54:33 -08001442 * In most cases the pmd is the only mapping of the page as we
1443 * break COW for the mlock() -- see gup_flags |= FOLL_WRITE for
1444 * writable private mappings in populate_vma_page_range().
1445 *
1446 * The only scenario when we have the page shared here is if we
1447 * mlocking read-only mapping shared over fork(). We skip
1448 * mlocking such pages.
Kirill A. Shutemov9a73f612016-07-26 15:25:53 -07001449 *
1450 * For file THP:
1451 *
1452 * We can expect PageDoubleMap() to be stable under page lock:
1453 * for file pages we set it in page_add_file_rmap(), which
1454 * requires page to be locked.
Kirill A. Shutemove90309c2016-01-15 16:54:33 -08001455 */
Kirill A. Shutemov9a73f612016-07-26 15:25:53 -07001456
1457 if (PageAnon(page) && compound_mapcount(page) != 1)
1458 goto skip_mlock;
1459 if (PageDoubleMap(page) || !page->mapping)
1460 goto skip_mlock;
1461 if (!trylock_page(page))
1462 goto skip_mlock;
1463 lru_add_drain();
1464 if (page->mapping && !PageDoubleMap(page))
1465 mlock_vma_page(page);
1466 unlock_page(page);
David Rientjesb676b292012-10-08 16:34:03 -07001467 }
Kirill A. Shutemov9a73f612016-07-26 15:25:53 -07001468skip_mlock:
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001469 page += (addr & ~HPAGE_PMD_MASK) >> PAGE_SHIFT;
Sasha Levin309381fea2014-01-23 15:52:54 -08001470 VM_BUG_ON_PAGE(!PageCompound(page), page);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001471 if (flags & FOLL_GET)
Kirill A. Shutemovddc58f22016-01-15 16:52:56 -08001472 get_page(page);
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001473
1474out:
1475 return page;
1476}
1477
Mel Gormand10e63f2012-10-25 14:16:31 +02001478/* NUMA hinting page fault entry point for trans huge pmds */
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001479int do_huge_pmd_numa_page(struct fault_env *fe, pmd_t pmd)
Mel Gormand10e63f2012-10-25 14:16:31 +02001480{
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001481 struct vm_area_struct *vma = fe->vma;
Mel Gormanb8916632013-10-07 11:28:44 +01001482 struct anon_vma *anon_vma = NULL;
Mel Gormanb32967f2012-11-19 12:35:47 +00001483 struct page *page;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001484 unsigned long haddr = fe->address & HPAGE_PMD_MASK;
Mel Gorman8191acb2013-10-07 11:28:45 +01001485 int page_nid = -1, this_nid = numa_node_id();
Peter Zijlstra90572892013-10-07 11:29:20 +01001486 int target_nid, last_cpupid = -1;
Mel Gorman8191acb2013-10-07 11:28:45 +01001487 bool page_locked;
1488 bool migrated = false;
Mel Gormanb191f9b2015-03-25 15:55:40 -07001489 bool was_writable;
Peter Zijlstra6688cc02013-10-07 11:29:24 +01001490 int flags = 0;
Mel Gormand10e63f2012-10-25 14:16:31 +02001491
Mel Gormanc0e7cad2015-02-12 14:58:41 -08001492 /* A PROT_NONE fault should not end up here */
1493 BUG_ON(!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)));
1494
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001495 fe->ptl = pmd_lock(vma->vm_mm, fe->pmd);
1496 if (unlikely(!pmd_same(pmd, *fe->pmd)))
Mel Gormand10e63f2012-10-25 14:16:31 +02001497 goto out_unlock;
1498
Mel Gormande466bd2013-12-18 17:08:42 -08001499 /*
1500 * If there are potential migrations, wait for completion and retry
1501 * without disrupting NUMA hinting information. Do not relock and
1502 * check_same as the page may no longer be mapped.
1503 */
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001504 if (unlikely(pmd_trans_migrating(*fe->pmd))) {
1505 page = pmd_page(*fe->pmd);
1506 spin_unlock(fe->ptl);
Mel Gorman5d833062015-02-12 14:58:16 -08001507 wait_on_page_locked(page);
Mel Gormande466bd2013-12-18 17:08:42 -08001508 goto out;
1509 }
1510
Mel Gormand10e63f2012-10-25 14:16:31 +02001511 page = pmd_page(pmd);
Mel Gormana1a46182013-10-07 11:28:50 +01001512 BUG_ON(is_huge_zero_page(page));
Mel Gorman8191acb2013-10-07 11:28:45 +01001513 page_nid = page_to_nid(page);
Peter Zijlstra90572892013-10-07 11:29:20 +01001514 last_cpupid = page_cpupid_last(page);
Mel Gorman03c5a6e2012-11-02 14:52:48 +00001515 count_vm_numa_event(NUMA_HINT_FAULTS);
Rik van Riel04bb2f92013-10-07 11:29:36 +01001516 if (page_nid == this_nid) {
Mel Gorman03c5a6e2012-11-02 14:52:48 +00001517 count_vm_numa_event(NUMA_HINT_FAULTS_LOCAL);
Rik van Riel04bb2f92013-10-07 11:29:36 +01001518 flags |= TNF_FAULT_LOCAL;
1519 }
Mel Gorman4daae3b2012-11-02 11:33:45 +00001520
Mel Gormanbea66fb2015-03-25 15:55:37 -07001521 /* See similar comment in do_numa_page for explanation */
1522 if (!(vma->vm_flags & VM_WRITE))
Peter Zijlstra6688cc02013-10-07 11:29:24 +01001523 flags |= TNF_NO_GROUP;
1524
1525 /*
Mel Gormanff9042b2013-10-07 11:28:43 +01001526 * Acquire the page lock to serialise THP migrations but avoid dropping
1527 * page_table_lock if at all possible
1528 */
Mel Gormanb8916632013-10-07 11:28:44 +01001529 page_locked = trylock_page(page);
1530 target_nid = mpol_misplaced(page, vma, haddr);
1531 if (target_nid == -1) {
1532 /* If the page was locked, there are no parallel migrations */
Mel Gormana54a4072013-10-07 11:28:46 +01001533 if (page_locked)
Mel Gormanb8916632013-10-07 11:28:44 +01001534 goto clear_pmdnuma;
Mel Gorman2b4847e2013-12-18 17:08:32 -08001535 }
Mel Gorman4daae3b2012-11-02 11:33:45 +00001536
Mel Gormande466bd2013-12-18 17:08:42 -08001537 /* Migration could have started since the pmd_trans_migrating check */
Mel Gorman2b4847e2013-12-18 17:08:32 -08001538 if (!page_locked) {
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001539 spin_unlock(fe->ptl);
Mel Gormanb8916632013-10-07 11:28:44 +01001540 wait_on_page_locked(page);
Mel Gormana54a4072013-10-07 11:28:46 +01001541 page_nid = -1;
Mel Gormanb8916632013-10-07 11:28:44 +01001542 goto out;
1543 }
1544
Mel Gorman2b4847e2013-12-18 17:08:32 -08001545 /*
1546 * Page is misplaced. Page lock serialises migrations. Acquire anon_vma
1547 * to serialises splits
1548 */
Mel Gormanb8916632013-10-07 11:28:44 +01001549 get_page(page);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001550 spin_unlock(fe->ptl);
Mel Gormanb8916632013-10-07 11:28:44 +01001551 anon_vma = page_lock_anon_vma_read(page);
Peter Zijlstracbee9f82012-10-25 14:16:43 +02001552
Peter Zijlstrac69307d2013-10-07 11:28:41 +01001553 /* Confirm the PMD did not change while page_table_lock was released */
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001554 spin_lock(fe->ptl);
1555 if (unlikely(!pmd_same(pmd, *fe->pmd))) {
Mel Gormanb32967f2012-11-19 12:35:47 +00001556 unlock_page(page);
1557 put_page(page);
Mel Gormana54a4072013-10-07 11:28:46 +01001558 page_nid = -1;
Mel Gormanb32967f2012-11-19 12:35:47 +00001559 goto out_unlock;
1560 }
Mel Gormanff9042b2013-10-07 11:28:43 +01001561
Mel Gormanc3a489c2013-12-18 17:08:38 -08001562 /* Bail if we fail to protect against THP splits for any reason */
1563 if (unlikely(!anon_vma)) {
1564 put_page(page);
1565 page_nid = -1;
1566 goto clear_pmdnuma;
1567 }
1568
Mel Gormana54a4072013-10-07 11:28:46 +01001569 /*
1570 * Migrate the THP to the requested node, returns with page unlocked
Mel Gorman8a0516e2015-02-12 14:58:22 -08001571 * and access rights restored.
Mel Gormana54a4072013-10-07 11:28:46 +01001572 */
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001573 spin_unlock(fe->ptl);
1574 migrated = migrate_misplaced_transhuge_page(vma->vm_mm, vma,
1575 fe->pmd, pmd, fe->address, page, target_nid);
Peter Zijlstra6688cc02013-10-07 11:29:24 +01001576 if (migrated) {
1577 flags |= TNF_MIGRATED;
Mel Gorman8191acb2013-10-07 11:28:45 +01001578 page_nid = target_nid;
Mel Gorman074c2382015-03-25 15:55:42 -07001579 } else
1580 flags |= TNF_MIGRATE_FAIL;
Mel Gormanb32967f2012-11-19 12:35:47 +00001581
Mel Gorman8191acb2013-10-07 11:28:45 +01001582 goto out;
Mel Gorman4daae3b2012-11-02 11:33:45 +00001583clear_pmdnuma:
Mel Gormana54a4072013-10-07 11:28:46 +01001584 BUG_ON(!PageLocked(page));
Mel Gormanb191f9b2015-03-25 15:55:40 -07001585 was_writable = pmd_write(pmd);
Mel Gorman4d942462015-02-12 14:58:28 -08001586 pmd = pmd_modify(pmd, vma->vm_page_prot);
Mel Gormanb7b04002015-03-25 15:55:45 -07001587 pmd = pmd_mkyoung(pmd);
Mel Gormanb191f9b2015-03-25 15:55:40 -07001588 if (was_writable)
1589 pmd = pmd_mkwrite(pmd);
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001590 set_pmd_at(vma->vm_mm, haddr, fe->pmd, pmd);
1591 update_mmu_cache_pmd(vma, fe->address, fe->pmd);
Mel Gormana54a4072013-10-07 11:28:46 +01001592 unlock_page(page);
Mel Gormand10e63f2012-10-25 14:16:31 +02001593out_unlock:
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001594 spin_unlock(fe->ptl);
Mel Gormanb8916632013-10-07 11:28:44 +01001595
1596out:
1597 if (anon_vma)
1598 page_unlock_anon_vma_read(anon_vma);
1599
Mel Gorman8191acb2013-10-07 11:28:45 +01001600 if (page_nid != -1)
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07001601 task_numa_fault(last_cpupid, page_nid, HPAGE_PMD_NR, fe->flags);
Mel Gorman8191acb2013-10-07 11:28:45 +01001602
Mel Gormand10e63f2012-10-25 14:16:31 +02001603 return 0;
1604}
1605
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001606int madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
1607 pmd_t *pmd, unsigned long addr, unsigned long next)
1608
1609{
1610 spinlock_t *ptl;
1611 pmd_t orig_pmd;
1612 struct page *page;
1613 struct mm_struct *mm = tlb->mm;
1614 int ret = 0;
1615
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001616 ptl = pmd_trans_huge_lock(pmd, vma);
1617 if (!ptl)
Linus Torvalds25eedab2016-01-17 18:33:15 -08001618 goto out_unlocked;
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001619
1620 orig_pmd = *pmd;
1621 if (is_huge_zero_pmd(orig_pmd)) {
1622 ret = 1;
1623 goto out;
1624 }
1625
1626 page = pmd_page(orig_pmd);
1627 /*
1628 * If other processes are mapping this page, we couldn't discard
1629 * the page unless they all do MADV_FREE so let's skip the page.
1630 */
1631 if (page_mapcount(page) != 1)
1632 goto out;
1633
1634 if (!trylock_page(page))
1635 goto out;
1636
1637 /*
1638 * If user want to discard part-pages of THP, split it so MADV_FREE
1639 * will deactivate only them.
1640 */
1641 if (next - addr != HPAGE_PMD_SIZE) {
1642 get_page(page);
1643 spin_unlock(ptl);
Huang Ying9818b8c2016-07-14 12:07:12 -07001644 split_huge_page(page);
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001645 put_page(page);
1646 unlock_page(page);
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08001647 goto out_unlocked;
1648 }
1649
1650 if (PageDirty(page))
1651 ClearPageDirty(page);
1652 unlock_page(page);
1653
1654 if (PageActive(page))
1655 deactivate_page(page);
1656
1657 if (pmd_young(orig_pmd) || pmd_dirty(orig_pmd)) {
1658 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
1659 tlb->fullmm);
1660 orig_pmd = pmd_mkold(orig_pmd);
1661 orig_pmd = pmd_mkclean(orig_pmd);
1662
1663 set_pmd_at(mm, addr, pmd, orig_pmd);
1664 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1665 }
1666 ret = 1;
1667out:
1668 spin_unlock(ptl);
1669out_unlocked:
1670 return ret;
1671}
1672
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001673int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
Shaohua Lif21760b2012-01-12 17:19:16 -08001674 pmd_t *pmd, unsigned long addr)
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001675{
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001676 pmd_t orig_pmd;
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001677 spinlock_t *ptl;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001678
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001679 ptl = __pmd_trans_huge_lock(pmd, vma);
1680 if (!ptl)
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001681 return 0;
1682 /*
1683 * For architectures like ppc64 we look at deposited pgtable
1684 * when calling pmdp_huge_get_and_clear. So do the
1685 * pgtable_trans_huge_withdraw after finishing pmdp related
1686 * operations.
1687 */
1688 orig_pmd = pmdp_huge_get_and_clear_full(tlb->mm, addr, pmd,
1689 tlb->fullmm);
1690 tlb_remove_pmd_tlb_entry(tlb, pmd, addr);
1691 if (vma_is_dax(vma)) {
1692 spin_unlock(ptl);
1693 if (is_huge_zero_pmd(orig_pmd))
Kirill A. Shutemovaa88b682016-04-28 16:18:27 -07001694 tlb_remove_page(tlb, pmd_page(orig_pmd));
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001695 } else if (is_huge_zero_pmd(orig_pmd)) {
1696 pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
1697 atomic_long_dec(&tlb->mm->nr_ptes);
1698 spin_unlock(ptl);
Kirill A. Shutemovaa88b682016-04-28 16:18:27 -07001699 tlb_remove_page(tlb, pmd_page(orig_pmd));
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001700 } else {
1701 struct page *page = pmd_page(orig_pmd);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08001702 page_remove_rmap(page, true);
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001703 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001704 VM_BUG_ON_PAGE(!PageHead(page), page);
Kirill A. Shutemovb5072382016-07-26 15:25:34 -07001705 if (PageAnon(page)) {
1706 pgtable_t pgtable;
1707 pgtable = pgtable_trans_huge_withdraw(tlb->mm, pmd);
1708 pte_free(tlb->mm, pgtable);
1709 atomic_long_dec(&tlb->mm->nr_ptes);
1710 add_mm_counter(tlb->mm, MM_ANONPAGES, -HPAGE_PMD_NR);
1711 } else {
1712 add_mm_counter(tlb->mm, MM_FILEPAGES, -HPAGE_PMD_NR);
1713 }
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001714 spin_unlock(ptl);
Aneesh Kumar K.Ve77b0852016-07-26 15:24:12 -07001715 tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001716 }
Kirill A. Shutemovda146762015-09-08 14:59:31 -07001717 return 1;
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -08001718}
1719
Hugh Dickinsbf8616d2016-05-19 17:12:54 -07001720bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001721 unsigned long new_addr, unsigned long old_end,
1722 pmd_t *old_pmd, pmd_t *new_pmd)
1723{
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001724 spinlock_t *old_ptl, *new_ptl;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001725 pmd_t pmd;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001726 struct mm_struct *mm = vma->vm_mm;
1727
1728 if ((old_addr & ~HPAGE_PMD_MASK) ||
1729 (new_addr & ~HPAGE_PMD_MASK) ||
Hugh Dickinsbf8616d2016-05-19 17:12:54 -07001730 old_end - old_addr < HPAGE_PMD_SIZE)
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001731 return false;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001732
1733 /*
1734 * The destination pmd shouldn't be established, free_pgtables()
1735 * should have release it.
1736 */
1737 if (WARN_ON(!pmd_none(*new_pmd))) {
1738 VM_BUG_ON(pmd_trans_huge(*new_pmd));
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001739 return false;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001740 }
1741
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001742 /*
1743 * We don't have to worry about the ordering of src and dst
1744 * ptlocks because exclusive mmap_sem prevents deadlock.
1745 */
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001746 old_ptl = __pmd_trans_huge_lock(old_pmd, vma);
1747 if (old_ptl) {
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001748 new_ptl = pmd_lockptr(mm, new_pmd);
1749 if (new_ptl != old_ptl)
1750 spin_lock_nested(new_ptl, SINGLE_DEPTH_NESTING);
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -07001751 pmd = pmdp_huge_get_and_clear(mm, old_addr, old_pmd);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001752 VM_BUG_ON(!pmd_none(*new_pmd));
Kirill A. Shutemov35928062013-12-12 17:12:33 -08001753
Kirill A. Shutemov69a8ec22016-02-17 13:11:12 -08001754 if (pmd_move_must_withdraw(new_ptl, old_ptl) &&
1755 vma_is_anonymous(vma)) {
Aneesh Kumar K.Vb3084f42014-01-13 11:34:24 +05301756 pgtable_t pgtable;
Kirill A. Shutemov35928062013-12-12 17:12:33 -08001757 pgtable = pgtable_trans_huge_withdraw(mm, old_pmd);
1758 pgtable_trans_huge_deposit(mm, new_pmd, pgtable);
Kirill A. Shutemov35928062013-12-12 17:12:33 -08001759 }
Aneesh Kumar K.Vb3084f42014-01-13 11:34:24 +05301760 set_pmd_at(mm, new_addr, new_pmd, pmd_mksoft_dirty(pmd));
1761 if (new_ptl != old_ptl)
1762 spin_unlock(new_ptl);
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001763 spin_unlock(old_ptl);
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001764 return true;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001765 }
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001766 return false;
Andrea Arcangeli37a1c492011-10-31 17:08:30 -07001767}
1768
Mel Gormanf123d742013-10-07 11:28:49 +01001769/*
1770 * Returns
1771 * - 0 if PMD could not be locked
1772 * - 1 if PMD was locked but protections unchange and TLB flush unnecessary
1773 * - HPAGE_PMD_NR is protections changed and TLB flush necessary
1774 */
Johannes Weinercd7548a2011-01-13 15:47:04 -08001775int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
Mel Gormane944fd62015-02-12 14:58:35 -08001776 unsigned long addr, pgprot_t newprot, int prot_numa)
Johannes Weinercd7548a2011-01-13 15:47:04 -08001777{
1778 struct mm_struct *mm = vma->vm_mm;
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001779 spinlock_t *ptl;
Johannes Weinercd7548a2011-01-13 15:47:04 -08001780 int ret = 0;
1781
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001782 ptl = __pmd_trans_huge_lock(pmd, vma);
1783 if (ptl) {
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001784 pmd_t entry;
Mel Gormanb191f9b2015-03-25 15:55:40 -07001785 bool preserve_write = prot_numa && pmd_write(*pmd);
Mel Gormanba68bc02015-03-07 15:20:48 +00001786 ret = 1;
Mel Gormane944fd62015-02-12 14:58:35 -08001787
1788 /*
1789 * Avoid trapping faults against the zero page. The read-only
1790 * data is likely to be read-cached on the local CPU and
1791 * local/remote hits to the zero page are not interesting.
1792 */
1793 if (prot_numa && is_huge_zero_pmd(*pmd)) {
1794 spin_unlock(ptl);
Mel Gormanba68bc02015-03-07 15:20:48 +00001795 return ret;
Mel Gormane944fd62015-02-12 14:58:35 -08001796 }
1797
Mel Gorman10c10452015-02-12 14:58:44 -08001798 if (!prot_numa || !pmd_protnone(*pmd)) {
Aneesh Kumar K.V8809aa22015-06-24 16:57:44 -07001799 entry = pmdp_huge_get_and_clear_notify(mm, addr, pmd);
Mel Gorman10c10452015-02-12 14:58:44 -08001800 entry = pmd_modify(entry, newprot);
Mel Gormanb191f9b2015-03-25 15:55:40 -07001801 if (preserve_write)
1802 entry = pmd_mkwrite(entry);
Mel Gorman10c10452015-02-12 14:58:44 -08001803 ret = HPAGE_PMD_NR;
1804 set_pmd_at(mm, addr, pmd, entry);
Kirill A. Shutemovb237ade2016-07-26 15:25:45 -07001805 BUG_ON(vma_is_anonymous(vma) && !preserve_write &&
1806 pmd_write(entry));
Mel Gorman10c10452015-02-12 14:58:44 -08001807 }
Kirill A. Shutemovbf929152013-11-14 14:30:54 -08001808 spin_unlock(ptl);
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001809 }
Johannes Weinercd7548a2011-01-13 15:47:04 -08001810
1811 return ret;
1812}
1813
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001814/*
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001815 * Returns true if a given pmd maps a thp, false otherwise.
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001816 *
Kirill A. Shutemov4b471e82016-01-15 16:53:39 -08001817 * Note that if it returns true, this routine returns without unlocking page
1818 * table lock. So callers must unlock it.
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001819 */
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001820spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma)
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001821{
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001822 spinlock_t *ptl;
1823 ptl = pmd_lock(vma->vm_mm, pmd);
Dan Williams5c7fb562016-01-15 16:56:52 -08001824 if (likely(pmd_trans_huge(*pmd) || pmd_devmap(*pmd)))
Kirill A. Shutemovb6ec57f2016-01-21 16:40:25 -08001825 return ptl;
1826 spin_unlock(ptl);
1827 return NULL;
Naoya Horiguchi025c5b22012-03-21 16:33:57 -07001828}
1829
Vlastimil Babka9050d7e2014-03-03 15:38:27 -08001830#define VM_NO_THP (VM_SPECIAL | VM_HUGETLB | VM_SHARED | VM_MAYSHARE)
Andrea Arcangeli78f11a22011-04-27 15:26:45 -07001831
Andrea Arcangeli60ab3242011-01-13 15:47:18 -08001832int hugepage_madvise(struct vm_area_struct *vma,
1833 unsigned long *vm_flags, int advice)
Andrea Arcangeli0af4e982011-01-13 15:46:55 -08001834{
Andrea Arcangelia664b2d2011-01-13 15:47:17 -08001835 switch (advice) {
1836 case MADV_HUGEPAGE:
Alex Thorlton1e1836e2014-04-07 15:37:09 -07001837#ifdef CONFIG_S390
1838 /*
1839 * qemu blindly sets MADV_HUGEPAGE on all allocations, but s390
1840 * can't handle this properly after s390_enable_sie, so we simply
1841 * ignore the madvise to prevent qemu from causing a SIGSEGV.
1842 */
1843 if (mm_has_pgste(vma->vm_mm))
1844 return 0;
1845#endif
Andrea Arcangelia664b2d2011-01-13 15:47:17 -08001846 /*
1847 * Be somewhat over-protective like KSM for now!
1848 */
Jason J. Herne1a763612015-11-20 15:57:04 -08001849 if (*vm_flags & VM_NO_THP)
Andrea Arcangelia664b2d2011-01-13 15:47:17 -08001850 return -EINVAL;
1851 *vm_flags &= ~VM_NOHUGEPAGE;
1852 *vm_flags |= VM_HUGEPAGE;
Andrea Arcangeli60ab3242011-01-13 15:47:18 -08001853 /*
1854 * If the vma become good for khugepaged to scan,
1855 * register it here without waiting a page fault that
1856 * may not happen any time soon.
1857 */
David Rientjes6d50e602014-10-29 14:50:31 -07001858 if (unlikely(khugepaged_enter_vma_merge(vma, *vm_flags)))
Andrea Arcangeli60ab3242011-01-13 15:47:18 -08001859 return -ENOMEM;
Andrea Arcangelia664b2d2011-01-13 15:47:17 -08001860 break;
1861 case MADV_NOHUGEPAGE:
1862 /*
1863 * Be somewhat over-protective like KSM for now!
1864 */
Jason J. Herne1a763612015-11-20 15:57:04 -08001865 if (*vm_flags & VM_NO_THP)
Andrea Arcangelia664b2d2011-01-13 15:47:17 -08001866 return -EINVAL;
1867 *vm_flags &= ~VM_HUGEPAGE;
1868 *vm_flags |= VM_NOHUGEPAGE;
Andrea Arcangeli60ab3242011-01-13 15:47:18 -08001869 /*
1870 * Setting VM_NOHUGEPAGE will prevent khugepaged from scanning
1871 * this vma even if we leave the mm registered in khugepaged if
1872 * it got registered before VM_NOHUGEPAGE was set.
1873 */
Andrea Arcangelia664b2d2011-01-13 15:47:17 -08001874 break;
1875 }
Andrea Arcangeli0af4e982011-01-13 15:46:55 -08001876
1877 return 0;
1878}
1879
Andrea Arcangeliba761492011-01-13 15:46:58 -08001880static int __init khugepaged_slab_init(void)
1881{
1882 mm_slot_cache = kmem_cache_create("khugepaged_mm_slot",
1883 sizeof(struct mm_slot),
1884 __alignof__(struct mm_slot), 0, NULL);
1885 if (!mm_slot_cache)
1886 return -ENOMEM;
1887
1888 return 0;
1889}
1890
Kirill A. Shutemov65ebb642015-04-15 16:14:20 -07001891static void __init khugepaged_slab_exit(void)
1892{
1893 kmem_cache_destroy(mm_slot_cache);
1894}
1895
Andrea Arcangeliba761492011-01-13 15:46:58 -08001896static inline struct mm_slot *alloc_mm_slot(void)
1897{
1898 if (!mm_slot_cache) /* initialization failed */
1899 return NULL;
1900 return kmem_cache_zalloc(mm_slot_cache, GFP_KERNEL);
1901}
1902
1903static inline void free_mm_slot(struct mm_slot *mm_slot)
1904{
1905 kmem_cache_free(mm_slot_cache, mm_slot);
1906}
1907
Andrea Arcangeliba761492011-01-13 15:46:58 -08001908static struct mm_slot *get_mm_slot(struct mm_struct *mm)
1909{
1910 struct mm_slot *mm_slot;
Andrea Arcangeliba761492011-01-13 15:46:58 -08001911
Sasha Levinb67bfe02013-02-27 17:06:00 -08001912 hash_for_each_possible(mm_slots_hash, mm_slot, hash, (unsigned long)mm)
Andrea Arcangeliba761492011-01-13 15:46:58 -08001913 if (mm == mm_slot->mm)
1914 return mm_slot;
Sasha Levin43b5fbb2013-02-22 16:32:27 -08001915
Andrea Arcangeliba761492011-01-13 15:46:58 -08001916 return NULL;
1917}
1918
1919static void insert_to_mm_slots_hash(struct mm_struct *mm,
1920 struct mm_slot *mm_slot)
1921{
Andrea Arcangeliba761492011-01-13 15:46:58 -08001922 mm_slot->mm = mm;
Sasha Levin43b5fbb2013-02-22 16:32:27 -08001923 hash_add(mm_slots_hash, &mm_slot->hash, (long)mm);
Andrea Arcangeliba761492011-01-13 15:46:58 -08001924}
1925
1926static inline int khugepaged_test_exit(struct mm_struct *mm)
1927{
1928 return atomic_read(&mm->mm_users) == 0;
1929}
1930
1931int __khugepaged_enter(struct mm_struct *mm)
1932{
1933 struct mm_slot *mm_slot;
1934 int wakeup;
1935
1936 mm_slot = alloc_mm_slot();
1937 if (!mm_slot)
1938 return -ENOMEM;
1939
1940 /* __khugepaged_exit() must not run from under us */
Sasha Levin96dad672014-10-09 15:28:39 -07001941 VM_BUG_ON_MM(khugepaged_test_exit(mm), mm);
Andrea Arcangeliba761492011-01-13 15:46:58 -08001942 if (unlikely(test_and_set_bit(MMF_VM_HUGEPAGE, &mm->flags))) {
1943 free_mm_slot(mm_slot);
1944 return 0;
1945 }
1946
1947 spin_lock(&khugepaged_mm_lock);
1948 insert_to_mm_slots_hash(mm, mm_slot);
1949 /*
1950 * Insert just behind the scanning cursor, to let the area settle
1951 * down a little.
1952 */
1953 wakeup = list_empty(&khugepaged_scan.mm_head);
1954 list_add_tail(&mm_slot->mm_node, &khugepaged_scan.mm_head);
1955 spin_unlock(&khugepaged_mm_lock);
1956
1957 atomic_inc(&mm->mm_count);
1958 if (wakeup)
1959 wake_up_interruptible(&khugepaged_wait);
1960
1961 return 0;
1962}
1963
David Rientjes6d50e602014-10-29 14:50:31 -07001964int khugepaged_enter_vma_merge(struct vm_area_struct *vma,
1965 unsigned long vm_flags)
Andrea Arcangeliba761492011-01-13 15:46:58 -08001966{
1967 unsigned long hstart, hend;
1968 if (!vma->anon_vma)
1969 /*
1970 * Not yet faulted in so we will register later in the
1971 * page fault if needed.
1972 */
1973 return 0;
Konstantin Khlebnikov3486b852016-04-28 16:18:32 -07001974 if (vma->vm_ops || (vm_flags & VM_NO_THP))
Andrea Arcangeliba761492011-01-13 15:46:58 -08001975 /* khugepaged not yet working on file or special mappings */
1976 return 0;
Andrea Arcangeliba761492011-01-13 15:46:58 -08001977 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
1978 hend = vma->vm_end & HPAGE_PMD_MASK;
1979 if (hstart < hend)
David Rientjes6d50e602014-10-29 14:50:31 -07001980 return khugepaged_enter(vma, vm_flags);
Andrea Arcangeliba761492011-01-13 15:46:58 -08001981 return 0;
1982}
1983
1984void __khugepaged_exit(struct mm_struct *mm)
1985{
1986 struct mm_slot *mm_slot;
1987 int free = 0;
1988
1989 spin_lock(&khugepaged_mm_lock);
1990 mm_slot = get_mm_slot(mm);
1991 if (mm_slot && khugepaged_scan.mm_slot != mm_slot) {
Sasha Levin43b5fbb2013-02-22 16:32:27 -08001992 hash_del(&mm_slot->hash);
Andrea Arcangeliba761492011-01-13 15:46:58 -08001993 list_del(&mm_slot->mm_node);
1994 free = 1;
1995 }
Chris Wrightd788e802011-07-25 17:12:14 -07001996 spin_unlock(&khugepaged_mm_lock);
Andrea Arcangeliba761492011-01-13 15:46:58 -08001997
1998 if (free) {
Andrea Arcangeliba761492011-01-13 15:46:58 -08001999 clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
2000 free_mm_slot(mm_slot);
2001 mmdrop(mm);
2002 } else if (mm_slot) {
Andrea Arcangeliba761492011-01-13 15:46:58 -08002003 /*
2004 * This is required to serialize against
2005 * khugepaged_test_exit() (which is guaranteed to run
2006 * under mmap sem read mode). Stop here (after we
2007 * return all pagetables will be destroyed) until
2008 * khugepaged has finished working on the pagetables
2009 * under the mmap_sem.
2010 */
2011 down_write(&mm->mmap_sem);
2012 up_write(&mm->mmap_sem);
Chris Wrightd788e802011-07-25 17:12:14 -07002013 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002014}
2015
2016static void release_pte_page(struct page *page)
2017{
2018 /* 0 stands for page_is_file_cache(page) == false */
2019 dec_zone_page_state(page, NR_ISOLATED_ANON + 0);
2020 unlock_page(page);
2021 putback_lru_page(page);
2022}
2023
2024static void release_pte_pages(pte_t *pte, pte_t *_pte)
2025{
2026 while (--_pte >= pte) {
2027 pte_t pteval = *_pte;
Ebru Akagunduzca0984c2015-04-14 15:45:24 -07002028 if (!pte_none(pteval) && !is_zero_pfn(pte_pfn(pteval)))
Andrea Arcangeliba761492011-01-13 15:46:58 -08002029 release_pte_page(pte_page(pteval));
2030 }
2031}
2032
Andrea Arcangeliba761492011-01-13 15:46:58 -08002033static int __collapse_huge_page_isolate(struct vm_area_struct *vma,
2034 unsigned long address,
2035 pte_t *pte)
2036{
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002037 struct page *page = NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002038 pte_t *_pte;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002039 int none_or_zero = 0, result = 0;
Ebru Akagunduz10359212015-02-11 15:28:28 -08002040 bool referenced = false, writable = false;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002041
Andrea Arcangeliba761492011-01-13 15:46:58 -08002042 for (_pte = pte; _pte < pte+HPAGE_PMD_NR;
2043 _pte++, address += PAGE_SIZE) {
2044 pte_t pteval = *_pte;
Minchan Kim47aee4d2015-10-22 13:32:19 -07002045 if (pte_none(pteval) || (pte_present(pteval) &&
2046 is_zero_pfn(pte_pfn(pteval)))) {
Andrea Arcangelic1294d02015-09-04 15:46:27 -07002047 if (!userfaultfd_armed(vma) &&
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002048 ++none_or_zero <= khugepaged_max_ptes_none) {
Andrea Arcangeliba761492011-01-13 15:46:58 -08002049 continue;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002050 } else {
2051 result = SCAN_EXCEED_NONE_PTE;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002052 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002053 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002054 }
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002055 if (!pte_present(pteval)) {
2056 result = SCAN_PTE_NON_PRESENT;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002057 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002058 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002059 page = vm_normal_page(vma, address, pteval);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002060 if (unlikely(!page)) {
2061 result = SCAN_PAGE_NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002062 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002063 }
Bob Liu344aa352012-12-11 16:00:34 -08002064
Sasha Levin309381fea2014-01-23 15:52:54 -08002065 VM_BUG_ON_PAGE(PageCompound(page), page);
2066 VM_BUG_ON_PAGE(!PageAnon(page), page);
2067 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002068
Andrea Arcangeliba761492011-01-13 15:46:58 -08002069 /*
2070 * We can do it before isolate_lru_page because the
2071 * page can't be freed from under us. NOTE: PG_lock
2072 * is needed to serialize against split_huge_page
2073 * when invoked from the VM.
2074 */
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002075 if (!trylock_page(page)) {
2076 result = SCAN_PAGE_LOCK;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002077 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002078 }
Ebru Akagunduz10359212015-02-11 15:28:28 -08002079
2080 /*
2081 * cannot use mapcount: can't collapse if there's a gup pin.
2082 * The page must only be referenced by the scanned process
2083 * and page swap cache.
2084 */
2085 if (page_count(page) != 1 + !!PageSwapCache(page)) {
2086 unlock_page(page);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002087 result = SCAN_PAGE_COUNT;
Ebru Akagunduz10359212015-02-11 15:28:28 -08002088 goto out;
2089 }
2090 if (pte_write(pteval)) {
2091 writable = true;
2092 } else {
Andrea Arcangeli6d0a07e2016-05-12 15:42:25 -07002093 if (PageSwapCache(page) &&
2094 !reuse_swap_page(page, NULL)) {
Ebru Akagunduz10359212015-02-11 15:28:28 -08002095 unlock_page(page);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002096 result = SCAN_SWAP_CACHE_PAGE;
Ebru Akagunduz10359212015-02-11 15:28:28 -08002097 goto out;
2098 }
2099 /*
2100 * Page is not in the swap cache. It can be collapsed
2101 * into a THP.
2102 */
2103 }
2104
Andrea Arcangeliba761492011-01-13 15:46:58 -08002105 /*
2106 * Isolate the page to avoid collapsing an hugepage
2107 * currently in use by the VM.
2108 */
2109 if (isolate_lru_page(page)) {
2110 unlock_page(page);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002111 result = SCAN_DEL_PAGE_LRU;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002112 goto out;
2113 }
2114 /* 0 stands for page_is_file_cache(page) == false */
2115 inc_zone_page_state(page, NR_ISOLATED_ANON + 0);
Sasha Levin309381fea2014-01-23 15:52:54 -08002116 VM_BUG_ON_PAGE(!PageLocked(page), page);
2117 VM_BUG_ON_PAGE(PageLRU(page), page);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002118
2119 /* If there is no mapped pte young don't collapse the page */
Vladimir Davydov33c3fc72015-09-09 15:35:45 -07002120 if (pte_young(pteval) ||
2121 page_is_young(page) || PageReferenced(page) ||
Andrea Arcangeli8ee53822011-01-13 15:47:10 -08002122 mmu_notifier_test_young(vma->vm_mm, address))
Ebru Akagunduz10359212015-02-11 15:28:28 -08002123 referenced = true;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002124 }
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002125 if (likely(writable)) {
2126 if (likely(referenced)) {
2127 result = SCAN_SUCCEED;
yalin wang16fd0fe2016-01-21 16:40:30 -08002128 trace_mm_collapse_huge_page_isolate(page, none_or_zero,
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002129 referenced, writable, result);
2130 return 1;
2131 }
2132 } else {
2133 result = SCAN_PAGE_RO;
2134 }
2135
Andrea Arcangeliba761492011-01-13 15:46:58 -08002136out:
Bob Liu344aa352012-12-11 16:00:34 -08002137 release_pte_pages(pte, _pte);
yalin wang16fd0fe2016-01-21 16:40:30 -08002138 trace_mm_collapse_huge_page_isolate(page, none_or_zero,
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002139 referenced, writable, result);
Bob Liu344aa352012-12-11 16:00:34 -08002140 return 0;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002141}
2142
2143static void __collapse_huge_page_copy(pte_t *pte, struct page *page,
2144 struct vm_area_struct *vma,
2145 unsigned long address,
2146 spinlock_t *ptl)
2147{
2148 pte_t *_pte;
2149 for (_pte = pte; _pte < pte+HPAGE_PMD_NR; _pte++) {
2150 pte_t pteval = *_pte;
2151 struct page *src_page;
2152
Ebru Akagunduzca0984c2015-04-14 15:45:24 -07002153 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
Andrea Arcangeliba761492011-01-13 15:46:58 -08002154 clear_user_highpage(page, address);
2155 add_mm_counter(vma->vm_mm, MM_ANONPAGES, 1);
Ebru Akagunduzca0984c2015-04-14 15:45:24 -07002156 if (is_zero_pfn(pte_pfn(pteval))) {
2157 /*
2158 * ptl mostly unnecessary.
2159 */
2160 spin_lock(ptl);
2161 /*
2162 * paravirt calls inside pte_clear here are
2163 * superfluous.
2164 */
2165 pte_clear(vma->vm_mm, address, _pte);
2166 spin_unlock(ptl);
2167 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002168 } else {
2169 src_page = pte_page(pteval);
2170 copy_user_highpage(page, src_page, address, vma);
Sasha Levin309381fea2014-01-23 15:52:54 -08002171 VM_BUG_ON_PAGE(page_mapcount(src_page) != 1, src_page);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002172 release_pte_page(src_page);
2173 /*
2174 * ptl mostly unnecessary, but preempt has to
2175 * be disabled to update the per-cpu stats
2176 * inside page_remove_rmap().
2177 */
2178 spin_lock(ptl);
2179 /*
2180 * paravirt calls inside pte_clear here are
2181 * superfluous.
2182 */
2183 pte_clear(vma->vm_mm, address, _pte);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08002184 page_remove_rmap(src_page, false);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002185 spin_unlock(ptl);
2186 free_page_and_swap_cache(src_page);
2187 }
2188
2189 address += PAGE_SIZE;
2190 page++;
2191 }
2192}
2193
Xiao Guangrong26234f32012-10-08 16:29:51 -07002194static void khugepaged_alloc_sleep(void)
2195{
Petr Mladekbde43c62015-09-08 15:04:05 -07002196 DEFINE_WAIT(wait);
2197
2198 add_wait_queue(&khugepaged_wait, &wait);
2199 freezable_schedule_timeout_interruptible(
2200 msecs_to_jiffies(khugepaged_alloc_sleep_millisecs));
2201 remove_wait_queue(&khugepaged_wait, &wait);
Xiao Guangrong26234f32012-10-08 16:29:51 -07002202}
2203
Bob Liu9f1b8682013-11-12 15:07:37 -08002204static int khugepaged_node_load[MAX_NUMNODES];
2205
David Rientjes14a4e212014-08-06 16:07:29 -07002206static bool khugepaged_scan_abort(int nid)
2207{
2208 int i;
2209
2210 /*
2211 * If zone_reclaim_mode is disabled, then no extra effort is made to
2212 * allocate memory locally.
2213 */
2214 if (!zone_reclaim_mode)
2215 return false;
2216
2217 /* If there is a count for this node already, it must be acceptable */
2218 if (khugepaged_node_load[nid])
2219 return false;
2220
2221 for (i = 0; i < MAX_NUMNODES; i++) {
2222 if (!khugepaged_node_load[i])
2223 continue;
2224 if (node_distance(nid, i) > RECLAIM_DISTANCE)
2225 return true;
2226 }
2227 return false;
2228}
2229
Xiao Guangrong26234f32012-10-08 16:29:51 -07002230#ifdef CONFIG_NUMA
Bob Liu9f1b8682013-11-12 15:07:37 -08002231static int khugepaged_find_target_node(void)
2232{
2233 static int last_khugepaged_target_node = NUMA_NO_NODE;
2234 int nid, target_node = 0, max_value = 0;
2235
2236 /* find first node with max normal pages hit */
2237 for (nid = 0; nid < MAX_NUMNODES; nid++)
2238 if (khugepaged_node_load[nid] > max_value) {
2239 max_value = khugepaged_node_load[nid];
2240 target_node = nid;
2241 }
2242
2243 /* do some balance if several nodes have the same hit record */
2244 if (target_node <= last_khugepaged_target_node)
2245 for (nid = last_khugepaged_target_node + 1; nid < MAX_NUMNODES;
2246 nid++)
2247 if (max_value == khugepaged_node_load[nid]) {
2248 target_node = nid;
2249 break;
2250 }
2251
2252 last_khugepaged_target_node = target_node;
2253 return target_node;
2254}
2255
Xiao Guangrong26234f32012-10-08 16:29:51 -07002256static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
2257{
2258 if (IS_ERR(*hpage)) {
2259 if (!*wait)
2260 return false;
2261
2262 *wait = false;
Xiao Guangronge3b41262012-10-08 16:32:57 -07002263 *hpage = NULL;
Xiao Guangrong26234f32012-10-08 16:29:51 -07002264 khugepaged_alloc_sleep();
2265 } else if (*hpage) {
2266 put_page(*hpage);
2267 *hpage = NULL;
2268 }
2269
2270 return true;
2271}
2272
Michal Hocko3b363692015-04-15 16:13:29 -07002273static struct page *
2274khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm,
Aaron Tomlind6669d62015-11-06 16:28:52 -08002275 unsigned long address, int node)
Xiao Guangrong26234f32012-10-08 16:29:51 -07002276{
Sasha Levin309381fea2014-01-23 15:52:54 -08002277 VM_BUG_ON_PAGE(*hpage, *hpage);
Vlastimil Babka8b164562014-10-09 15:27:00 -07002278
Xiao Guangrong26234f32012-10-08 16:29:51 -07002279 /*
Vlastimil Babka8b164562014-10-09 15:27:00 -07002280 * Before allocating the hugepage, release the mmap_sem read lock.
2281 * The allocation can take potentially a long time if it involves
2282 * sync compaction, and we do not need to hold the mmap_sem during
2283 * that. We will recheck the vma after taking it again in write mode.
Xiao Guangrong26234f32012-10-08 16:29:51 -07002284 */
2285 up_read(&mm->mmap_sem);
Vlastimil Babka8b164562014-10-09 15:27:00 -07002286
Vlastimil Babka96db8002015-09-08 15:03:50 -07002287 *hpage = __alloc_pages_node(node, gfp, HPAGE_PMD_ORDER);
Xiao Guangrong26234f32012-10-08 16:29:51 -07002288 if (unlikely(!*hpage)) {
2289 count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
2290 *hpage = ERR_PTR(-ENOMEM);
2291 return NULL;
2292 }
2293
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002294 prep_transhuge_page(*hpage);
Xiao Guangrong26234f32012-10-08 16:29:51 -07002295 count_vm_event(THP_COLLAPSE_ALLOC);
2296 return *hpage;
2297}
2298#else
Bob Liu9f1b8682013-11-12 15:07:37 -08002299static int khugepaged_find_target_node(void)
2300{
2301 return 0;
2302}
2303
Mel Gorman444eb2a42016-03-17 14:19:23 -07002304static inline struct page *alloc_khugepaged_hugepage(void)
Bob Liu10dc4152013-11-12 15:07:35 -08002305{
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002306 struct page *page;
2307
Mel Gorman444eb2a42016-03-17 14:19:23 -07002308 page = alloc_pages(alloc_hugepage_khugepaged_gfpmask(),
2309 HPAGE_PMD_ORDER);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08002310 if (page)
2311 prep_transhuge_page(page);
2312 return page;
Bob Liu10dc4152013-11-12 15:07:35 -08002313}
2314
Xiao Guangrong26234f32012-10-08 16:29:51 -07002315static struct page *khugepaged_alloc_hugepage(bool *wait)
2316{
2317 struct page *hpage;
2318
2319 do {
Mel Gorman444eb2a42016-03-17 14:19:23 -07002320 hpage = alloc_khugepaged_hugepage();
Xiao Guangrong26234f32012-10-08 16:29:51 -07002321 if (!hpage) {
2322 count_vm_event(THP_COLLAPSE_ALLOC_FAILED);
2323 if (!*wait)
2324 return NULL;
2325
2326 *wait = false;
2327 khugepaged_alloc_sleep();
2328 } else
2329 count_vm_event(THP_COLLAPSE_ALLOC);
2330 } while (unlikely(!hpage) && likely(khugepaged_enabled()));
2331
2332 return hpage;
2333}
2334
2335static bool khugepaged_prealloc_page(struct page **hpage, bool *wait)
2336{
2337 if (!*hpage)
2338 *hpage = khugepaged_alloc_hugepage(wait);
2339
2340 if (unlikely(!*hpage))
2341 return false;
2342
2343 return true;
2344}
2345
Michal Hocko3b363692015-04-15 16:13:29 -07002346static struct page *
2347khugepaged_alloc_page(struct page **hpage, gfp_t gfp, struct mm_struct *mm,
Aaron Tomlind6669d62015-11-06 16:28:52 -08002348 unsigned long address, int node)
Xiao Guangrong26234f32012-10-08 16:29:51 -07002349{
2350 up_read(&mm->mmap_sem);
2351 VM_BUG_ON(!*hpage);
Michal Hocko3b363692015-04-15 16:13:29 -07002352
Xiao Guangrong26234f32012-10-08 16:29:51 -07002353 return *hpage;
2354}
2355#endif
2356
Bob Liufa475e52012-12-11 16:00:39 -08002357static bool hugepage_vma_check(struct vm_area_struct *vma)
2358{
2359 if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
2360 (vma->vm_flags & VM_NOHUGEPAGE))
2361 return false;
Bob Liufa475e52012-12-11 16:00:39 -08002362 if (!vma->anon_vma || vma->vm_ops)
2363 return false;
2364 if (is_vma_temporary_stack(vma))
2365 return false;
Konstantin Khlebnikov3486b852016-04-28 16:18:32 -07002366 return !(vma->vm_flags & VM_NO_THP);
Bob Liufa475e52012-12-11 16:00:39 -08002367}
2368
Ebru Akagunduz8a966ed2016-07-26 15:25:03 -07002369/*
Ebru Akagunduz72695862016-07-26 15:25:06 -07002370 * If mmap_sem temporarily dropped, revalidate vma
2371 * before taking mmap_sem.
2372 * Return 0 if succeeds, otherwise return none-zero
2373 * value (scan code).
2374 */
2375
2376static int hugepage_vma_revalidate(struct mm_struct *mm, unsigned long address)
2377{
2378 struct vm_area_struct *vma;
2379 unsigned long hstart, hend;
2380
2381 if (unlikely(khugepaged_test_exit(mm)))
2382 return SCAN_ANY_PROCESS;
2383
2384 vma = find_vma(mm, address);
2385 if (!vma)
2386 return SCAN_VMA_NULL;
2387
2388 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
2389 hend = vma->vm_end & HPAGE_PMD_MASK;
2390 if (address < hstart || address + HPAGE_PMD_SIZE > hend)
2391 return SCAN_ADDRESS_RANGE;
2392 if (!hugepage_vma_check(vma))
2393 return SCAN_VMA_CHECK;
2394 return 0;
2395}
2396
2397/*
Ebru Akagunduz8a966ed2016-07-26 15:25:03 -07002398 * Bring missing pages in from swap, to complete THP collapse.
2399 * Only done if khugepaged_scan_pmd believes it is worthwhile.
2400 *
2401 * Called and returns without pte mapped or spinlocks held,
2402 * but with mmap_sem held to protect against vma changes.
2403 */
2404
Ebru Akagunduz72695862016-07-26 15:25:06 -07002405static bool __collapse_huge_page_swapin(struct mm_struct *mm,
Ebru Akagunduz8a966ed2016-07-26 15:25:03 -07002406 struct vm_area_struct *vma,
2407 unsigned long address, pmd_t *pmd)
2408{
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07002409 pte_t pteval;
Ebru Akagunduz8a966ed2016-07-26 15:25:03 -07002410 int swapped_in = 0, ret = 0;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07002411 struct fault_env fe = {
2412 .vma = vma,
2413 .address = address,
2414 .flags = FAULT_FLAG_ALLOW_RETRY,
2415 .pmd = pmd,
2416 };
Ebru Akagunduz8a966ed2016-07-26 15:25:03 -07002417
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07002418 fe.pte = pte_offset_map(pmd, address);
2419 for (; fe.address < address + HPAGE_PMD_NR*PAGE_SIZE;
2420 fe.pte++, fe.address += PAGE_SIZE) {
2421 pteval = *fe.pte;
Ebru Akagunduz8a966ed2016-07-26 15:25:03 -07002422 if (!is_swap_pte(pteval))
2423 continue;
2424 swapped_in++;
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07002425 ret = do_swap_page(&fe, pteval);
Ebru Akagunduz72695862016-07-26 15:25:06 -07002426 /* do_swap_page returns VM_FAULT_RETRY with released mmap_sem */
2427 if (ret & VM_FAULT_RETRY) {
2428 down_read(&mm->mmap_sem);
2429 /* vma is no longer available, don't continue to swapin */
2430 if (hugepage_vma_revalidate(mm, address))
2431 return false;
Kirill A. Shutemov1f52e672016-07-26 15:25:12 -07002432 /* check if the pmd is still valid */
2433 if (mm_find_pmd(mm, address) != pmd)
2434 return false;
Ebru Akagunduz72695862016-07-26 15:25:06 -07002435 }
Ebru Akagunduz8a966ed2016-07-26 15:25:03 -07002436 if (ret & VM_FAULT_ERROR) {
2437 trace_mm_collapse_huge_page_swapin(mm, swapped_in, 0);
Ebru Akagunduz72695862016-07-26 15:25:06 -07002438 return false;
Ebru Akagunduz8a966ed2016-07-26 15:25:03 -07002439 }
2440 /* pte is unmapped now, we need to map it */
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07002441 fe.pte = pte_offset_map(pmd, fe.address);
Ebru Akagunduz8a966ed2016-07-26 15:25:03 -07002442 }
Kirill A. Shutemovbae473a2016-07-26 15:25:20 -07002443 fe.pte--;
2444 pte_unmap(fe.pte);
Ebru Akagunduz8a966ed2016-07-26 15:25:03 -07002445 trace_mm_collapse_huge_page_swapin(mm, swapped_in, 1);
Ebru Akagunduz72695862016-07-26 15:25:06 -07002446 return true;
Ebru Akagunduz8a966ed2016-07-26 15:25:03 -07002447}
2448
Andrea Arcangeliba761492011-01-13 15:46:58 -08002449static void collapse_huge_page(struct mm_struct *mm,
Xiao Guangrong26234f32012-10-08 16:29:51 -07002450 unsigned long address,
2451 struct page **hpage,
2452 struct vm_area_struct *vma,
2453 int node)
Andrea Arcangeliba761492011-01-13 15:46:58 -08002454{
Andrea Arcangeliba761492011-01-13 15:46:58 -08002455 pmd_t *pmd, _pmd;
2456 pte_t *pte;
2457 pgtable_t pgtable;
2458 struct page *new_page;
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002459 spinlock_t *pmd_ptl, *pte_ptl;
Arnd Bergmann629d9d12016-01-18 21:50:26 +01002460 int isolated = 0, result = 0;
Johannes Weiner00501b52014-08-08 14:19:20 -07002461 struct mem_cgroup *memcg;
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07002462 unsigned long mmun_start; /* For mmu_notifiers */
2463 unsigned long mmun_end; /* For mmu_notifiers */
Michal Hocko3b363692015-04-15 16:13:29 -07002464 gfp_t gfp;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002465
2466 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
Andrea Arcangeli692e0b32011-05-24 17:12:14 -07002467
Michal Hocko3b363692015-04-15 16:13:29 -07002468 /* Only allocate from the target node */
Mel Gorman444eb2a42016-03-17 14:19:23 -07002469 gfp = alloc_hugepage_khugepaged_gfpmask() | __GFP_OTHER_NODE | __GFP_THISNODE;
Michal Hocko3b363692015-04-15 16:13:29 -07002470
Xiao Guangrong26234f32012-10-08 16:29:51 -07002471 /* release the mmap_sem read lock. */
Aaron Tomlind6669d62015-11-06 16:28:52 -08002472 new_page = khugepaged_alloc_page(hpage, gfp, mm, address, node);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002473 if (!new_page) {
2474 result = SCAN_ALLOC_HUGE_PAGE_FAIL;
2475 goto out_nolock;
2476 }
Andrea Arcangelice83d212011-01-13 15:47:06 -08002477
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08002478 if (unlikely(mem_cgroup_try_charge(new_page, mm, gfp, &memcg, true))) {
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002479 result = SCAN_CGROUP_CHARGE_FAIL;
2480 goto out_nolock;
2481 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002482
Ebru Akagunduz72695862016-07-26 15:25:06 -07002483 down_read(&mm->mmap_sem);
2484 result = hugepage_vma_revalidate(mm, address);
Ebru Akagunduz8024ee22016-07-26 15:25:09 -07002485 if (result) {
2486 mem_cgroup_cancel_charge(new_page, memcg, true);
2487 up_read(&mm->mmap_sem);
2488 goto out_nolock;
2489 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002490
Bob Liu62190492012-12-11 16:00:37 -08002491 pmd = mm_find_pmd(mm, address);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002492 if (!pmd) {
2493 result = SCAN_PMD_NULL;
Ebru Akagunduz8024ee22016-07-26 15:25:09 -07002494 mem_cgroup_cancel_charge(new_page, memcg, true);
2495 up_read(&mm->mmap_sem);
2496 goto out_nolock;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002497 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002498
Ebru Akagunduz72695862016-07-26 15:25:06 -07002499 /*
2500 * __collapse_huge_page_swapin always returns with mmap_sem locked.
2501 * If it fails, release mmap_sem and jump directly out.
2502 * Continuing to collapse causes inconsistency.
2503 */
2504 if (!__collapse_huge_page_swapin(mm, vma, address, pmd)) {
Ebru Akagunduz8024ee22016-07-26 15:25:09 -07002505 mem_cgroup_cancel_charge(new_page, memcg, true);
Ebru Akagunduz72695862016-07-26 15:25:06 -07002506 up_read(&mm->mmap_sem);
Ebru Akagunduz8024ee22016-07-26 15:25:09 -07002507 goto out_nolock;
Ebru Akagunduz72695862016-07-26 15:25:06 -07002508 }
2509
2510 up_read(&mm->mmap_sem);
2511 /*
2512 * Prevent all access to pagetables with the exception of
2513 * gup_fast later handled by the ptep_clear_flush and the VM
2514 * handled by the anon_vma lock + PG_lock.
2515 */
2516 down_write(&mm->mmap_sem);
2517 result = hugepage_vma_revalidate(mm, address);
2518 if (result)
2519 goto out;
Kirill A. Shutemov1f52e672016-07-26 15:25:12 -07002520 /* check if the pmd is still valid */
2521 if (mm_find_pmd(mm, address) != pmd)
2522 goto out;
Ebru Akagunduz8a966ed2016-07-26 15:25:03 -07002523
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +00002524 anon_vma_lock_write(vma->anon_vma);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002525
2526 pte = pte_offset_map(pmd, address);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002527 pte_ptl = pte_lockptr(mm, pmd);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002528
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07002529 mmun_start = address;
2530 mmun_end = address + HPAGE_PMD_SIZE;
2531 mmu_notifier_invalidate_range_start(mm, mmun_start, mmun_end);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002532 pmd_ptl = pmd_lock(mm, pmd); /* probably unnecessary */
Andrea Arcangeliba761492011-01-13 15:46:58 -08002533 /*
2534 * After this gup_fast can't run anymore. This also removes
2535 * any huge TLB entry from the CPU so we won't allow
2536 * huge and small TLB entries for the same virtual address
2537 * to avoid the risk of CPU bugs in that area.
2538 */
Aneesh Kumar K.V15a25b22015-06-24 16:57:39 -07002539 _pmd = pmdp_collapse_flush(vma, address, pmd);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002540 spin_unlock(pmd_ptl);
Sagi Grimberg2ec74c32012-10-08 16:33:33 -07002541 mmu_notifier_invalidate_range_end(mm, mmun_start, mmun_end);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002542
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002543 spin_lock(pte_ptl);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002544 isolated = __collapse_huge_page_isolate(vma, address, pte);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002545 spin_unlock(pte_ptl);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002546
2547 if (unlikely(!isolated)) {
Johannes Weiner453c7192011-01-20 14:44:18 -08002548 pte_unmap(pte);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002549 spin_lock(pmd_ptl);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002550 BUG_ON(!pmd_none(*pmd));
Aneesh Kumar K.V7c342512013-05-24 15:55:21 -07002551 /*
2552 * We can only use set_pmd_at when establishing
2553 * hugepmds and never for establishing regular pmds that
2554 * points to regular pagetables. Use pmd_populate for that
2555 */
2556 pmd_populate(mm, pmd, pmd_pgtable(_pmd));
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002557 spin_unlock(pmd_ptl);
Konstantin Khlebnikov08b52702013-02-22 16:34:40 -08002558 anon_vma_unlock_write(vma->anon_vma);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002559 result = SCAN_FAIL;
Andrea Arcangelice83d212011-01-13 15:47:06 -08002560 goto out;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002561 }
2562
2563 /*
2564 * All pages are isolated and locked so anon_vma rmap
2565 * can't run anymore.
2566 */
Konstantin Khlebnikov08b52702013-02-22 16:34:40 -08002567 anon_vma_unlock_write(vma->anon_vma);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002568
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002569 __collapse_huge_page_copy(pte, new_page, vma, address, pte_ptl);
Johannes Weiner453c7192011-01-20 14:44:18 -08002570 pte_unmap(pte);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002571 __SetPageUptodate(new_page);
2572 pgtable = pmd_pgtable(_pmd);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002573
Kirill A. Shutemov31223592013-09-12 15:14:01 -07002574 _pmd = mk_huge_pmd(new_page, vma->vm_page_prot);
2575 _pmd = maybe_pmd_mkwrite(pmd_mkdirty(_pmd), vma);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002576
2577 /*
2578 * spin_lock() below is not the equivalent of smp_wmb(), so
2579 * this is needed to avoid the copy_huge_page writes to become
2580 * visible after the set_pmd_at() write.
2581 */
2582 smp_wmb();
2583
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002584 spin_lock(pmd_ptl);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002585 BUG_ON(!pmd_none(*pmd));
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -08002586 page_add_new_anon_rmap(new_page, vma, address, true);
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08002587 mem_cgroup_commit_charge(new_page, memcg, false, true);
Johannes Weiner00501b52014-08-08 14:19:20 -07002588 lru_cache_add_active_or_unevictable(new_page, vma);
Aneesh Kumar K.Vfce144b2013-06-05 17:14:06 -07002589 pgtable_trans_huge_deposit(mm, pmd, pgtable);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002590 set_pmd_at(mm, address, pmd, _pmd);
David Millerb113da62012-10-08 16:34:25 -07002591 update_mmu_cache_pmd(vma, address, pmd);
Kirill A. Shutemovc4088eb2013-11-14 14:31:04 -08002592 spin_unlock(pmd_ptl);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002593
2594 *hpage = NULL;
Xiao Guangrong420256ef2012-10-08 16:29:49 -07002595
Andrea Arcangeliba761492011-01-13 15:46:58 -08002596 khugepaged_pages_collapsed++;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002597 result = SCAN_SUCCEED;
Andrea Arcangelice83d212011-01-13 15:47:06 -08002598out_up_write:
Andrea Arcangeliba761492011-01-13 15:46:58 -08002599 up_write(&mm->mmap_sem);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002600out_nolock:
2601 trace_mm_collapse_huge_page(mm, isolated, result);
2602 return;
Andrea Arcangelice83d212011-01-13 15:47:06 -08002603out:
Kirill A. Shutemovf627c2f2016-01-15 16:52:20 -08002604 mem_cgroup_cancel_charge(new_page, memcg, true);
Andrea Arcangelice83d212011-01-13 15:47:06 -08002605 goto out_up_write;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002606}
2607
2608static int khugepaged_scan_pmd(struct mm_struct *mm,
2609 struct vm_area_struct *vma,
2610 unsigned long address,
2611 struct page **hpage)
2612{
Andrea Arcangeliba761492011-01-13 15:46:58 -08002613 pmd_t *pmd;
2614 pte_t *pte, *_pte;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002615 int ret = 0, none_or_zero = 0, result = 0;
2616 struct page *page = NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002617 unsigned long _address;
2618 spinlock_t *ptl;
Ebru Akagunduz70652f62016-07-26 15:24:59 -07002619 int node = NUMA_NO_NODE, unmapped = 0;
Ebru Akagunduz10359212015-02-11 15:28:28 -08002620 bool writable = false, referenced = false;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002621
2622 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
2623
Bob Liu62190492012-12-11 16:00:37 -08002624 pmd = mm_find_pmd(mm, address);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002625 if (!pmd) {
2626 result = SCAN_PMD_NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002627 goto out;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002628 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002629
Bob Liu9f1b8682013-11-12 15:07:37 -08002630 memset(khugepaged_node_load, 0, sizeof(khugepaged_node_load));
Andrea Arcangeliba761492011-01-13 15:46:58 -08002631 pte = pte_offset_map_lock(mm, pmd, address, &ptl);
2632 for (_address = address, _pte = pte; _pte < pte+HPAGE_PMD_NR;
2633 _pte++, _address += PAGE_SIZE) {
2634 pte_t pteval = *_pte;
Ebru Akagunduz70652f62016-07-26 15:24:59 -07002635 if (is_swap_pte(pteval)) {
2636 if (++unmapped <= khugepaged_max_ptes_swap) {
2637 continue;
2638 } else {
2639 result = SCAN_EXCEED_SWAP_PTE;
2640 goto out_unmap;
2641 }
2642 }
Ebru Akagunduzca0984c2015-04-14 15:45:24 -07002643 if (pte_none(pteval) || is_zero_pfn(pte_pfn(pteval))) {
Andrea Arcangelic1294d02015-09-04 15:46:27 -07002644 if (!userfaultfd_armed(vma) &&
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002645 ++none_or_zero <= khugepaged_max_ptes_none) {
Andrea Arcangeliba761492011-01-13 15:46:58 -08002646 continue;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002647 } else {
2648 result = SCAN_EXCEED_NONE_PTE;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002649 goto out_unmap;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002650 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002651 }
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002652 if (!pte_present(pteval)) {
2653 result = SCAN_PTE_NON_PRESENT;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002654 goto out_unmap;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002655 }
Ebru Akagunduz10359212015-02-11 15:28:28 -08002656 if (pte_write(pteval))
2657 writable = true;
2658
Andrea Arcangeliba761492011-01-13 15:46:58 -08002659 page = vm_normal_page(vma, _address, pteval);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002660 if (unlikely(!page)) {
2661 result = SCAN_PAGE_NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002662 goto out_unmap;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002663 }
Kirill A. Shutemovb1caa952016-01-15 16:52:39 -08002664
2665 /* TODO: teach khugepaged to collapse THP mapped with pte */
2666 if (PageCompound(page)) {
2667 result = SCAN_PAGE_COMPOUND;
2668 goto out_unmap;
2669 }
2670
Andi Kleen5c4b4be2011-03-04 17:36:32 -08002671 /*
Bob Liu9f1b8682013-11-12 15:07:37 -08002672 * Record which node the original page is from and save this
2673 * information to khugepaged_node_load[].
2674 * Khupaged will allocate hugepage from the node has the max
2675 * hit record.
Andi Kleen5c4b4be2011-03-04 17:36:32 -08002676 */
Bob Liu9f1b8682013-11-12 15:07:37 -08002677 node = page_to_nid(page);
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002678 if (khugepaged_scan_abort(node)) {
2679 result = SCAN_SCAN_ABORT;
David Rientjes14a4e212014-08-06 16:07:29 -07002680 goto out_unmap;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002681 }
Bob Liu9f1b8682013-11-12 15:07:37 -08002682 khugepaged_node_load[node]++;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002683 if (!PageLRU(page)) {
Kirill A. Shutemov0fda2782016-03-25 14:22:20 -07002684 result = SCAN_PAGE_LRU;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002685 goto out_unmap;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002686 }
2687 if (PageLocked(page)) {
2688 result = SCAN_PAGE_LOCK;
2689 goto out_unmap;
2690 }
2691 if (!PageAnon(page)) {
2692 result = SCAN_PAGE_ANON;
2693 goto out_unmap;
2694 }
2695
Ebru Akagunduz10359212015-02-11 15:28:28 -08002696 /*
2697 * cannot use mapcount: can't collapse if there's a gup pin.
2698 * The page must only be referenced by the scanned process
2699 * and page swap cache.
2700 */
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002701 if (page_count(page) != 1 + !!PageSwapCache(page)) {
2702 result = SCAN_PAGE_COUNT;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002703 goto out_unmap;
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002704 }
Vladimir Davydov33c3fc72015-09-09 15:35:45 -07002705 if (pte_young(pteval) ||
2706 page_is_young(page) || PageReferenced(page) ||
Andrea Arcangeli8ee53822011-01-13 15:47:10 -08002707 mmu_notifier_test_young(vma->vm_mm, address))
Ebru Akagunduz10359212015-02-11 15:28:28 -08002708 referenced = true;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002709 }
Ebru Akagunduz7d2eba02016-01-14 15:22:19 -08002710 if (writable) {
2711 if (referenced) {
2712 result = SCAN_SUCCEED;
2713 ret = 1;
2714 } else {
2715 result = SCAN_NO_REFERENCED_PAGE;
2716 }
2717 } else {
2718 result = SCAN_PAGE_RO;
2719 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002720out_unmap:
2721 pte_unmap_unlock(pte, ptl);
Bob Liu9f1b8682013-11-12 15:07:37 -08002722 if (ret) {
2723 node = khugepaged_find_target_node();
Andrea Arcangelice83d212011-01-13 15:47:06 -08002724 /* collapse_huge_page will return with the mmap_sem released */
Andi Kleen5c4b4be2011-03-04 17:36:32 -08002725 collapse_huge_page(mm, address, hpage, vma, node);
Bob Liu9f1b8682013-11-12 15:07:37 -08002726 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002727out:
yalin wang16fd0fe2016-01-21 16:40:30 -08002728 trace_mm_khugepaged_scan_pmd(mm, page, writable, referenced,
Ebru Akagunduz70652f62016-07-26 15:24:59 -07002729 none_or_zero, result, unmapped);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002730 return ret;
2731}
2732
2733static void collect_mm_slot(struct mm_slot *mm_slot)
2734{
2735 struct mm_struct *mm = mm_slot->mm;
2736
Hugh Dickinsb9980cd2012-02-08 17:13:40 -08002737 VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
Andrea Arcangeliba761492011-01-13 15:46:58 -08002738
2739 if (khugepaged_test_exit(mm)) {
2740 /* free mm_slot */
Sasha Levin43b5fbb2013-02-22 16:32:27 -08002741 hash_del(&mm_slot->hash);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002742 list_del(&mm_slot->mm_node);
2743
2744 /*
2745 * Not strictly needed because the mm exited already.
2746 *
2747 * clear_bit(MMF_VM_HUGEPAGE, &mm->flags);
2748 */
2749
2750 /* khugepaged_mm_lock actually not necessary for the below */
2751 free_mm_slot(mm_slot);
2752 mmdrop(mm);
2753 }
2754}
2755
2756static unsigned int khugepaged_scan_mm_slot(unsigned int pages,
2757 struct page **hpage)
H Hartley Sweeten2f1da642011-10-31 17:09:25 -07002758 __releases(&khugepaged_mm_lock)
2759 __acquires(&khugepaged_mm_lock)
Andrea Arcangeliba761492011-01-13 15:46:58 -08002760{
2761 struct mm_slot *mm_slot;
2762 struct mm_struct *mm;
2763 struct vm_area_struct *vma;
2764 int progress = 0;
2765
2766 VM_BUG_ON(!pages);
Hugh Dickinsb9980cd2012-02-08 17:13:40 -08002767 VM_BUG_ON(NR_CPUS != 1 && !spin_is_locked(&khugepaged_mm_lock));
Andrea Arcangeliba761492011-01-13 15:46:58 -08002768
2769 if (khugepaged_scan.mm_slot)
2770 mm_slot = khugepaged_scan.mm_slot;
2771 else {
2772 mm_slot = list_entry(khugepaged_scan.mm_head.next,
2773 struct mm_slot, mm_node);
2774 khugepaged_scan.address = 0;
2775 khugepaged_scan.mm_slot = mm_slot;
2776 }
2777 spin_unlock(&khugepaged_mm_lock);
2778
2779 mm = mm_slot->mm;
2780 down_read(&mm->mmap_sem);
2781 if (unlikely(khugepaged_test_exit(mm)))
2782 vma = NULL;
2783 else
2784 vma = find_vma(mm, khugepaged_scan.address);
2785
2786 progress++;
2787 for (; vma; vma = vma->vm_next) {
2788 unsigned long hstart, hend;
2789
2790 cond_resched();
2791 if (unlikely(khugepaged_test_exit(mm))) {
2792 progress++;
2793 break;
2794 }
Bob Liufa475e52012-12-11 16:00:39 -08002795 if (!hugepage_vma_check(vma)) {
2796skip:
Andrea Arcangeliba761492011-01-13 15:46:58 -08002797 progress++;
2798 continue;
2799 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002800 hstart = (vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK;
2801 hend = vma->vm_end & HPAGE_PMD_MASK;
Andrea Arcangelia7d6e4e2011-02-15 19:02:45 +01002802 if (hstart >= hend)
2803 goto skip;
2804 if (khugepaged_scan.address > hend)
2805 goto skip;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002806 if (khugepaged_scan.address < hstart)
2807 khugepaged_scan.address = hstart;
Andrea Arcangelia7d6e4e2011-02-15 19:02:45 +01002808 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002809
2810 while (khugepaged_scan.address < hend) {
2811 int ret;
2812 cond_resched();
2813 if (unlikely(khugepaged_test_exit(mm)))
2814 goto breakouterloop;
2815
2816 VM_BUG_ON(khugepaged_scan.address < hstart ||
2817 khugepaged_scan.address + HPAGE_PMD_SIZE >
2818 hend);
2819 ret = khugepaged_scan_pmd(mm, vma,
2820 khugepaged_scan.address,
2821 hpage);
2822 /* move to next address */
2823 khugepaged_scan.address += HPAGE_PMD_SIZE;
2824 progress += HPAGE_PMD_NR;
2825 if (ret)
2826 /* we released mmap_sem so break loop */
2827 goto breakouterloop_mmap_sem;
2828 if (progress >= pages)
2829 goto breakouterloop;
2830 }
2831 }
2832breakouterloop:
2833 up_read(&mm->mmap_sem); /* exit_mmap will destroy ptes after this */
2834breakouterloop_mmap_sem:
2835
2836 spin_lock(&khugepaged_mm_lock);
Andrea Arcangelia7d6e4e2011-02-15 19:02:45 +01002837 VM_BUG_ON(khugepaged_scan.mm_slot != mm_slot);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002838 /*
2839 * Release the current mm_slot if this mm is about to die, or
2840 * if we scanned all vmas of this mm.
2841 */
2842 if (khugepaged_test_exit(mm) || !vma) {
2843 /*
2844 * Make sure that if mm_users is reaching zero while
2845 * khugepaged runs here, khugepaged_exit will find
2846 * mm_slot not pointing to the exiting mm.
2847 */
2848 if (mm_slot->mm_node.next != &khugepaged_scan.mm_head) {
2849 khugepaged_scan.mm_slot = list_entry(
2850 mm_slot->mm_node.next,
2851 struct mm_slot, mm_node);
2852 khugepaged_scan.address = 0;
2853 } else {
2854 khugepaged_scan.mm_slot = NULL;
2855 khugepaged_full_scans++;
2856 }
2857
2858 collect_mm_slot(mm_slot);
2859 }
2860
2861 return progress;
2862}
2863
2864static int khugepaged_has_work(void)
2865{
2866 return !list_empty(&khugepaged_scan.mm_head) &&
2867 khugepaged_enabled();
2868}
2869
2870static int khugepaged_wait_event(void)
2871{
2872 return !list_empty(&khugepaged_scan.mm_head) ||
Xiao Guangrong2017c0b2012-10-08 16:29:44 -07002873 kthread_should_stop();
Andrea Arcangeliba761492011-01-13 15:46:58 -08002874}
2875
Xiao Guangrongd5169042012-10-08 16:29:48 -07002876static void khugepaged_do_scan(void)
2877{
2878 struct page *hpage = NULL;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002879 unsigned int progress = 0, pass_through_head = 0;
2880 unsigned int pages = khugepaged_pages_to_scan;
Xiao Guangrongd5169042012-10-08 16:29:48 -07002881 bool wait = true;
Andrea Arcangeliba761492011-01-13 15:46:58 -08002882
2883 barrier(); /* write khugepaged_pages_to_scan to local stack */
2884
2885 while (progress < pages) {
Xiao Guangrong26234f32012-10-08 16:29:51 -07002886 if (!khugepaged_prealloc_page(&hpage, &wait))
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08002887 break;
Xiao Guangrong26234f32012-10-08 16:29:51 -07002888
Xiao Guangrong420256ef2012-10-08 16:29:49 -07002889 cond_resched();
Andrea Arcangeliba761492011-01-13 15:46:58 -08002890
Jiri Kosinacd092412015-06-24 16:56:07 -07002891 if (unlikely(kthread_should_stop() || try_to_freeze()))
Andrea Arcangeli878aee72011-01-13 15:47:10 -08002892 break;
2893
Andrea Arcangeliba761492011-01-13 15:46:58 -08002894 spin_lock(&khugepaged_mm_lock);
2895 if (!khugepaged_scan.mm_slot)
2896 pass_through_head++;
2897 if (khugepaged_has_work() &&
2898 pass_through_head < 2)
2899 progress += khugepaged_scan_mm_slot(pages - progress,
Xiao Guangrongd5169042012-10-08 16:29:48 -07002900 &hpage);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002901 else
2902 progress = pages;
2903 spin_unlock(&khugepaged_mm_lock);
2904 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002905
Xiao Guangrongd5169042012-10-08 16:29:48 -07002906 if (!IS_ERR_OR_NULL(hpage))
2907 put_page(hpage);
Andrea Arcangeli0bbbc0b2011-01-13 15:47:05 -08002908}
2909
David Rientjesf0508972016-05-20 16:58:56 -07002910static bool khugepaged_should_wakeup(void)
2911{
2912 return kthread_should_stop() ||
2913 time_after_eq(jiffies, khugepaged_sleep_expire);
2914}
2915
Xiao Guangrong2017c0b2012-10-08 16:29:44 -07002916static void khugepaged_wait_work(void)
2917{
Xiao Guangrong2017c0b2012-10-08 16:29:44 -07002918 if (khugepaged_has_work()) {
David Rientjesf0508972016-05-20 16:58:56 -07002919 const unsigned long scan_sleep_jiffies =
2920 msecs_to_jiffies(khugepaged_scan_sleep_millisecs);
2921
2922 if (!scan_sleep_jiffies)
Xiao Guangrong2017c0b2012-10-08 16:29:44 -07002923 return;
2924
David Rientjesf0508972016-05-20 16:58:56 -07002925 khugepaged_sleep_expire = jiffies + scan_sleep_jiffies;
Xiao Guangrong2017c0b2012-10-08 16:29:44 -07002926 wait_event_freezable_timeout(khugepaged_wait,
David Rientjesf0508972016-05-20 16:58:56 -07002927 khugepaged_should_wakeup(),
2928 scan_sleep_jiffies);
Xiao Guangrong2017c0b2012-10-08 16:29:44 -07002929 return;
2930 }
2931
2932 if (khugepaged_enabled())
2933 wait_event_freezable(khugepaged_wait, khugepaged_wait_event());
2934}
2935
Andrea Arcangeliba761492011-01-13 15:46:58 -08002936static int khugepaged(void *none)
2937{
2938 struct mm_slot *mm_slot;
2939
Andrea Arcangeli878aee72011-01-13 15:47:10 -08002940 set_freezable();
Dongsheng Yang8698a742014-03-11 18:09:12 +08002941 set_user_nice(current, MAX_NICE);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002942
Xiao Guangrongb7231782012-10-08 16:29:54 -07002943 while (!kthread_should_stop()) {
2944 khugepaged_do_scan();
2945 khugepaged_wait_work();
2946 }
Andrea Arcangeliba761492011-01-13 15:46:58 -08002947
2948 spin_lock(&khugepaged_mm_lock);
2949 mm_slot = khugepaged_scan.mm_slot;
2950 khugepaged_scan.mm_slot = NULL;
2951 if (mm_slot)
2952 collect_mm_slot(mm_slot);
2953 spin_unlock(&khugepaged_mm_lock);
Andrea Arcangeliba761492011-01-13 15:46:58 -08002954 return 0;
2955}
2956
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002957static void __split_huge_zero_page_pmd(struct vm_area_struct *vma,
2958 unsigned long haddr, pmd_t *pmd)
2959{
2960 struct mm_struct *mm = vma->vm_mm;
2961 pgtable_t pgtable;
2962 pmd_t _pmd;
2963 int i;
2964
2965 /* leave pmd empty until pte is filled */
2966 pmdp_huge_clear_flush_notify(vma, haddr, pmd);
2967
2968 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
2969 pmd_populate(mm, &_pmd, pgtable);
2970
2971 for (i = 0; i < HPAGE_PMD_NR; i++, haddr += PAGE_SIZE) {
2972 pte_t *pte, entry;
2973 entry = pfn_pte(my_zero_pfn(haddr), vma->vm_page_prot);
2974 entry = pte_mkspecial(entry);
2975 pte = pte_offset_map(&_pmd, haddr);
2976 VM_BUG_ON(!pte_none(*pte));
2977 set_pte_at(mm, haddr, pte, entry);
2978 pte_unmap(pte);
2979 }
2980 smp_wmb(); /* make pte visible before pmd */
2981 pmd_populate(mm, pmd, pgtable);
2982 put_huge_zero_page();
2983}
2984
2985static void __split_huge_pmd_locked(struct vm_area_struct *vma, pmd_t *pmd,
Kirill A. Shutemovba988282016-01-15 16:53:56 -08002986 unsigned long haddr, bool freeze)
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002987{
2988 struct mm_struct *mm = vma->vm_mm;
2989 struct page *page;
2990 pgtable_t pgtable;
2991 pmd_t _pmd;
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08002992 bool young, write, dirty;
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03002993 unsigned long addr;
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08002994 int i;
2995
2996 VM_BUG_ON(haddr & ~HPAGE_PMD_MASK);
2997 VM_BUG_ON_VMA(vma->vm_start > haddr, vma);
2998 VM_BUG_ON_VMA(vma->vm_end < haddr + HPAGE_PMD_SIZE, vma);
Dan Williams5c7fb562016-01-15 16:56:52 -08002999 VM_BUG_ON(!pmd_trans_huge(*pmd) && !pmd_devmap(*pmd));
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003000
3001 count_vm_event(THP_SPLIT_PMD);
3002
Kirill A. Shutemovd21b9e52016-07-26 15:25:37 -07003003 if (!vma_is_anonymous(vma)) {
3004 _pmd = pmdp_huge_clear_flush_notify(vma, haddr, pmd);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003005 if (is_huge_zero_pmd(_pmd))
3006 put_huge_zero_page();
Kirill A. Shutemovd21b9e52016-07-26 15:25:37 -07003007 if (vma_is_dax(vma))
3008 return;
3009 page = pmd_page(_pmd);
3010 if (!PageReferenced(page) && pmd_young(_pmd))
3011 SetPageReferenced(page);
3012 page_remove_rmap(page, true);
3013 put_page(page);
3014 add_mm_counter(mm, MM_FILEPAGES, -HPAGE_PMD_NR);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003015 return;
3016 } else if (is_huge_zero_pmd(*pmd)) {
3017 return __split_huge_zero_page_pmd(vma, haddr, pmd);
3018 }
3019
3020 page = pmd_page(*pmd);
3021 VM_BUG_ON_PAGE(!page_count(page), page);
Joonsoo Kimfe896d12016-03-17 14:19:26 -07003022 page_ref_add(page, HPAGE_PMD_NR - 1);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003023 write = pmd_write(*pmd);
3024 young = pmd_young(*pmd);
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08003025 dirty = pmd_dirty(*pmd);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003026
Aneesh Kumar K.Vc777e2a2016-02-09 06:50:31 +05303027 pmdp_huge_split_prepare(vma, haddr, pmd);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003028 pgtable = pgtable_trans_huge_withdraw(mm, pmd);
3029 pmd_populate(mm, &_pmd, pgtable);
3030
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03003031 for (i = 0, addr = haddr; i < HPAGE_PMD_NR; i++, addr += PAGE_SIZE) {
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003032 pte_t entry, *pte;
3033 /*
3034 * Note that NUMA hinting access restrictions are not
3035 * transferred to avoid any possibility of altering
3036 * permissions across VMAs.
3037 */
Kirill A. Shutemovba988282016-01-15 16:53:56 -08003038 if (freeze) {
3039 swp_entry_t swp_entry;
3040 swp_entry = make_migration_entry(page + i, write);
3041 entry = swp_entry_to_pte(swp_entry);
3042 } else {
3043 entry = mk_pte(page + i, vma->vm_page_prot);
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08003044 entry = maybe_mkwrite(entry, vma);
Kirill A. Shutemovba988282016-01-15 16:53:56 -08003045 if (!write)
3046 entry = pte_wrprotect(entry);
3047 if (!young)
3048 entry = pte_mkold(entry);
3049 }
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08003050 if (dirty)
3051 SetPageDirty(page + i);
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03003052 pte = pte_offset_map(&_pmd, addr);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003053 BUG_ON(!pte_none(*pte));
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03003054 set_pte_at(mm, addr, pte, entry);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003055 atomic_inc(&page[i]._mapcount);
3056 pte_unmap(pte);
3057 }
3058
3059 /*
3060 * Set PG_double_map before dropping compound_mapcount to avoid
3061 * false-negative page_mapped().
3062 */
3063 if (compound_mapcount(page) > 1 && !TestSetPageDoubleMap(page)) {
3064 for (i = 0; i < HPAGE_PMD_NR; i++)
3065 atomic_inc(&page[i]._mapcount);
3066 }
3067
3068 if (atomic_add_negative(-1, compound_mapcount_ptr(page))) {
3069 /* Last compound_mapcount is gone. */
Kirill A. Shutemov65c45372016-07-26 15:26:10 -07003070 __dec_zone_page_state(page, NR_ANON_THPS);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003071 if (TestClearPageDoubleMap(page)) {
3072 /* No need in mapcount reference anymore */
3073 for (i = 0; i < HPAGE_PMD_NR; i++)
3074 atomic_dec(&page[i]._mapcount);
3075 }
3076 }
3077
3078 smp_wmb(); /* make pte visible before pmd */
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003079 /*
3080 * Up to this point the pmd is present and huge and userland has the
3081 * whole access to the hugepage during the split (which happens in
3082 * place). If we overwrite the pmd with the not-huge version pointing
3083 * to the pte here (which of course we could if all CPUs were bug
3084 * free), userland could trigger a small page size TLB miss on the
3085 * small sized TLB while the hugepage TLB entry is still established in
3086 * the huge TLB. Some CPU doesn't like that.
3087 * See http://support.amd.com/us/Processor_TechDocs/41322.pdf, Erratum
3088 * 383 on page 93. Intel should be safe but is also warns that it's
3089 * only safe if the permission and cache attributes of the two entries
3090 * loaded in the two TLB is identical (which should be the case here).
3091 * But it is generally safer to never allow small and huge TLB entries
3092 * for the same virtual address to be loaded simultaneously. So instead
3093 * of doing "pmd_populate(); flush_pmd_tlb_range();" we first mark the
3094 * current pmd notpresent (atomically because here the pmd_trans_huge
3095 * and pmd_trans_splitting must remain set at all times on the pmd
3096 * until the split is complete for this pmd), then we flush the SMP TLB
3097 * and finally we write the non-huge version of the pmd entry with
3098 * pmd_populate.
3099 */
3100 pmdp_invalidate(vma, haddr, pmd);
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003101 pmd_populate(mm, pmd, pgtable);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003102
3103 if (freeze) {
Kirill A. Shutemov2ac015e2016-02-24 18:58:03 +03003104 for (i = 0; i < HPAGE_PMD_NR; i++) {
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003105 page_remove_rmap(page + i, false);
3106 put_page(page + i);
3107 }
3108 }
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003109}
3110
3111void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd,
Naoya Horiguchi33f47512016-07-14 12:07:32 -07003112 unsigned long address, bool freeze, struct page *page)
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003113{
3114 spinlock_t *ptl;
3115 struct mm_struct *mm = vma->vm_mm;
3116 unsigned long haddr = address & HPAGE_PMD_MASK;
3117
3118 mmu_notifier_invalidate_range_start(mm, haddr, haddr + HPAGE_PMD_SIZE);
3119 ptl = pmd_lock(mm, pmd);
Naoya Horiguchi33f47512016-07-14 12:07:32 -07003120
3121 /*
3122 * If caller asks to setup a migration entries, we need a page to check
3123 * pmd against. Otherwise we can end up replacing wrong page.
3124 */
3125 VM_BUG_ON(freeze && !page);
3126 if (page && page != pmd_page(*pmd))
3127 goto out;
3128
Dan Williams5c7fb562016-01-15 16:56:52 -08003129 if (pmd_trans_huge(*pmd)) {
Naoya Horiguchi33f47512016-07-14 12:07:32 -07003130 page = pmd_page(*pmd);
Dan Williams5c7fb562016-01-15 16:56:52 -08003131 if (PageMlocked(page))
Kirill A. Shutemov5f737712016-03-17 14:20:13 -07003132 clear_page_mlock(page);
Dan Williams5c7fb562016-01-15 16:56:52 -08003133 } else if (!pmd_devmap(*pmd))
Kirill A. Shutemove90309c2016-01-15 16:54:33 -08003134 goto out;
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003135 __split_huge_pmd_locked(vma, pmd, haddr, freeze);
Kirill A. Shutemove90309c2016-01-15 16:54:33 -08003136out:
Kirill A. Shutemoveef1b3b2016-01-15 16:53:53 -08003137 spin_unlock(ptl);
3138 mmu_notifier_invalidate_range_end(mm, haddr, haddr + HPAGE_PMD_SIZE);
3139}
3140
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003141void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address,
3142 bool freeze, struct page *page)
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003143{
Hugh Dickinsf72e7dc2014-06-23 13:22:05 -07003144 pgd_t *pgd;
3145 pud_t *pud;
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003146 pmd_t *pmd;
3147
Kirill A. Shutemov78ddc532016-01-15 16:52:42 -08003148 pgd = pgd_offset(vma->vm_mm, address);
Hugh Dickinsf72e7dc2014-06-23 13:22:05 -07003149 if (!pgd_present(*pgd))
3150 return;
3151
3152 pud = pud_offset(pgd, address);
3153 if (!pud_present(*pud))
3154 return;
3155
3156 pmd = pmd_offset(pud, address);
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003157
Naoya Horiguchi33f47512016-07-14 12:07:32 -07003158 __split_huge_pmd(vma, pmd, address, freeze, page);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003159}
3160
Kirill A. Shutemove1b99962015-09-08 14:58:37 -07003161void vma_adjust_trans_huge(struct vm_area_struct *vma,
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003162 unsigned long start,
3163 unsigned long end,
3164 long adjust_next)
3165{
3166 /*
3167 * If the new start address isn't hpage aligned and it could
3168 * previously contain an hugepage: check if we need to split
3169 * an huge pmd.
3170 */
3171 if (start & ~HPAGE_PMD_MASK &&
3172 (start & HPAGE_PMD_MASK) >= vma->vm_start &&
3173 (start & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003174 split_huge_pmd_address(vma, start, false, NULL);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003175
3176 /*
3177 * If the new end address isn't hpage aligned and it could
3178 * previously contain an hugepage: check if we need to split
3179 * an huge pmd.
3180 */
3181 if (end & ~HPAGE_PMD_MASK &&
3182 (end & HPAGE_PMD_MASK) >= vma->vm_start &&
3183 (end & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= vma->vm_end)
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003184 split_huge_pmd_address(vma, end, false, NULL);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003185
3186 /*
3187 * If we're also updating the vma->vm_next->vm_start, if the new
3188 * vm_next->vm_start isn't page aligned and it could previously
3189 * contain an hugepage: check if we need to split an huge pmd.
3190 */
3191 if (adjust_next > 0) {
3192 struct vm_area_struct *next = vma->vm_next;
3193 unsigned long nstart = next->vm_start;
3194 nstart += adjust_next << PAGE_SHIFT;
3195 if (nstart & ~HPAGE_PMD_MASK &&
3196 (nstart & HPAGE_PMD_MASK) >= next->vm_start &&
3197 (nstart & HPAGE_PMD_MASK) + HPAGE_PMD_SIZE <= next->vm_end)
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003198 split_huge_pmd_address(next, nstart, false, NULL);
Andrea Arcangeli94fcc582011-01-13 15:47:08 -08003199 }
3200}
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003201
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003202static void freeze_page(struct page *page)
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003203{
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003204 enum ttu_flags ttu_flags = TTU_IGNORE_MLOCK | TTU_IGNORE_ACCESS |
3205 TTU_RMAP_LOCKED;
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003206 int i, ret;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003207
3208 VM_BUG_ON_PAGE(!PageHead(page), page);
3209
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003210 if (PageAnon(page))
3211 ttu_flags |= TTU_MIGRATION;
3212
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003213 /* We only need TTU_SPLIT_HUGE_PMD once */
3214 ret = try_to_unmap(page, ttu_flags | TTU_SPLIT_HUGE_PMD);
3215 for (i = 1; !ret && i < HPAGE_PMD_NR; i++) {
3216 /* Cut short if the page is unmapped */
3217 if (page_count(page) == 1)
3218 return;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003219
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003220 ret = try_to_unmap(page + i, ttu_flags);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003221 }
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003222 VM_BUG_ON_PAGE(ret, page + i - 1);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003223}
3224
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003225static void unfreeze_page(struct page *page)
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003226{
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003227 int i;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003228
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003229 for (i = 0; i < HPAGE_PMD_NR; i++)
3230 remove_migration_ptes(page + i, page + i, true);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003231}
3232
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07003233static void __split_huge_page_tail(struct page *head, int tail,
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003234 struct lruvec *lruvec, struct list_head *list)
3235{
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003236 struct page *page_tail = head + tail;
3237
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07003238 VM_BUG_ON_PAGE(atomic_read(&page_tail->_mapcount) != -1, page_tail);
Joonsoo Kimfe896d12016-03-17 14:19:26 -07003239 VM_BUG_ON_PAGE(page_ref_count(page_tail) != 0, page_tail);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003240
3241 /*
Joonsoo Kim0139aa72016-05-19 17:10:49 -07003242 * tail_page->_refcount is zero and not changing from under us. But
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003243 * get_page_unless_zero() may be running from under us on the
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003244 * tail_page. If we used atomic_set() below instead of atomic_inc() or
3245 * atomic_add(), we would then run atomic_set() concurrently with
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003246 * get_page_unless_zero(), and atomic_set() is implemented in C not
3247 * using locked ops. spin_unlock on x86 sometime uses locked ops
3248 * because of PPro errata 66, 92, so unless somebody can guarantee
3249 * atomic_set() here would be safe on all archs (and not only on x86),
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003250 * it's safer to use atomic_inc()/atomic_add().
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003251 */
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003252 if (PageAnon(head)) {
3253 page_ref_inc(page_tail);
3254 } else {
3255 /* Additional pin to radix tree */
3256 page_ref_add(page_tail, 2);
3257 }
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003258
3259 page_tail->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
3260 page_tail->flags |= (head->flags &
3261 ((1L << PG_referenced) |
3262 (1L << PG_swapbacked) |
3263 (1L << PG_mlocked) |
3264 (1L << PG_uptodate) |
3265 (1L << PG_active) |
3266 (1L << PG_locked) |
Minchan Kimb8d3c4c2016-01-15 16:55:42 -08003267 (1L << PG_unevictable) |
3268 (1L << PG_dirty)));
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003269
3270 /*
3271 * After clearing PageTail the gup refcount can be released.
3272 * Page flags also must be visible before we make the page non-compound.
3273 */
3274 smp_wmb();
3275
3276 clear_compound_head(page_tail);
3277
3278 if (page_is_young(head))
3279 set_page_young(page_tail);
3280 if (page_is_idle(head))
3281 set_page_idle(page_tail);
3282
3283 /* ->mapping in first tail page is compound_mapcount */
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003284 VM_BUG_ON_PAGE(tail > 2 && page_tail->mapping != TAIL_MAPPING,
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003285 page_tail);
3286 page_tail->mapping = head->mapping;
3287
3288 page_tail->index = head->index + tail;
3289 page_cpupid_xchg_last(page_tail, page_cpupid_last(head));
3290 lru_add_page_tail(head, page_tail, lruvec, list);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003291}
3292
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003293static void __split_huge_page(struct page *page, struct list_head *list,
3294 unsigned long flags)
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003295{
3296 struct page *head = compound_head(page);
3297 struct zone *zone = page_zone(head);
3298 struct lruvec *lruvec;
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003299 pgoff_t end = -1;
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07003300 int i;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003301
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003302 lruvec = mem_cgroup_page_lruvec(head, zone);
3303
3304 /* complete memcg works before add pages to LRU */
3305 mem_cgroup_split_huge_fixup(head);
3306
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003307 if (!PageAnon(page))
3308 end = DIV_ROUND_UP(i_size_read(head->mapping->host), PAGE_SIZE);
3309
3310 for (i = HPAGE_PMD_NR - 1; i >= 1; i--) {
Kirill A. Shutemov8df651c2016-03-15 14:57:30 -07003311 __split_huge_page_tail(head, i, lruvec, list);
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003312 /* Some pages can be beyond i_size: drop them from page cache */
3313 if (head[i].index >= end) {
3314 __ClearPageDirty(head + i);
3315 __delete_from_page_cache(head + i, NULL);
3316 put_page(head + i);
3317 }
3318 }
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003319
3320 ClearPageCompound(head);
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003321 /* See comment in __split_huge_page_tail() */
3322 if (PageAnon(head)) {
3323 page_ref_inc(head);
3324 } else {
3325 /* Additional pin to radix tree */
3326 page_ref_add(head, 2);
3327 spin_unlock(&head->mapping->tree_lock);
3328 }
3329
3330 spin_unlock_irqrestore(&page_zone(head)->lru_lock, flags);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003331
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003332 unfreeze_page(head);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003333
3334 for (i = 0; i < HPAGE_PMD_NR; i++) {
3335 struct page *subpage = head + i;
3336 if (subpage == page)
3337 continue;
3338 unlock_page(subpage);
3339
3340 /*
3341 * Subpages may be freed if there wasn't any mapping
3342 * like if add_to_swap() is running on a lru page that
3343 * had its mapping zapped. And freeing these pages
3344 * requires taking the lru_lock so we do the put_page
3345 * of the tail pages after the split is complete.
3346 */
3347 put_page(subpage);
3348 }
3349}
3350
Kirill A. Shutemovb20ce5e2016-01-15 16:54:37 -08003351int total_mapcount(struct page *page)
3352{
Kirill A. Shutemovdd78fed2016-07-26 15:25:26 -07003353 int i, compound, ret;
Kirill A. Shutemovb20ce5e2016-01-15 16:54:37 -08003354
3355 VM_BUG_ON_PAGE(PageTail(page), page);
3356
3357 if (likely(!PageCompound(page)))
3358 return atomic_read(&page->_mapcount) + 1;
3359
Kirill A. Shutemovdd78fed2016-07-26 15:25:26 -07003360 compound = compound_mapcount(page);
Kirill A. Shutemovb20ce5e2016-01-15 16:54:37 -08003361 if (PageHuge(page))
Kirill A. Shutemovdd78fed2016-07-26 15:25:26 -07003362 return compound;
3363 ret = compound;
Kirill A. Shutemovb20ce5e2016-01-15 16:54:37 -08003364 for (i = 0; i < HPAGE_PMD_NR; i++)
3365 ret += atomic_read(&page[i]._mapcount) + 1;
Kirill A. Shutemovdd78fed2016-07-26 15:25:26 -07003366 /* File pages has compound_mapcount included in _mapcount */
3367 if (!PageAnon(page))
3368 return ret - compound * HPAGE_PMD_NR;
Kirill A. Shutemovb20ce5e2016-01-15 16:54:37 -08003369 if (PageDoubleMap(page))
3370 ret -= HPAGE_PMD_NR;
3371 return ret;
3372}
3373
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003374/*
Andrea Arcangeli6d0a07e2016-05-12 15:42:25 -07003375 * This calculates accurately how many mappings a transparent hugepage
3376 * has (unlike page_mapcount() which isn't fully accurate). This full
3377 * accuracy is primarily needed to know if copy-on-write faults can
3378 * reuse the page and change the mapping to read-write instead of
3379 * copying them. At the same time this returns the total_mapcount too.
3380 *
3381 * The function returns the highest mapcount any one of the subpages
3382 * has. If the return value is one, even if different processes are
3383 * mapping different subpages of the transparent hugepage, they can
3384 * all reuse it, because each process is reusing a different subpage.
3385 *
3386 * The total_mapcount is instead counting all virtual mappings of the
3387 * subpages. If the total_mapcount is equal to "one", it tells the
3388 * caller all mappings belong to the same "mm" and in turn the
3389 * anon_vma of the transparent hugepage can become the vma->anon_vma
3390 * local one as no other process may be mapping any of the subpages.
3391 *
3392 * It would be more accurate to replace page_mapcount() with
3393 * page_trans_huge_mapcount(), however we only use
3394 * page_trans_huge_mapcount() in the copy-on-write faults where we
3395 * need full accuracy to avoid breaking page pinning, because
3396 * page_trans_huge_mapcount() is slower than page_mapcount().
3397 */
3398int page_trans_huge_mapcount(struct page *page, int *total_mapcount)
3399{
3400 int i, ret, _total_mapcount, mapcount;
3401
3402 /* hugetlbfs shouldn't call it */
3403 VM_BUG_ON_PAGE(PageHuge(page), page);
3404
3405 if (likely(!PageTransCompound(page))) {
3406 mapcount = atomic_read(&page->_mapcount) + 1;
3407 if (total_mapcount)
3408 *total_mapcount = mapcount;
3409 return mapcount;
3410 }
3411
3412 page = compound_head(page);
3413
3414 _total_mapcount = ret = 0;
3415 for (i = 0; i < HPAGE_PMD_NR; i++) {
3416 mapcount = atomic_read(&page[i]._mapcount) + 1;
3417 ret = max(ret, mapcount);
3418 _total_mapcount += mapcount;
3419 }
3420 if (PageDoubleMap(page)) {
3421 ret -= 1;
3422 _total_mapcount -= HPAGE_PMD_NR;
3423 }
3424 mapcount = compound_mapcount(page);
3425 ret += mapcount;
3426 _total_mapcount += mapcount;
3427 if (total_mapcount)
3428 *total_mapcount = _total_mapcount;
3429 return ret;
3430}
3431
3432/*
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003433 * This function splits huge page into normal pages. @page can point to any
3434 * subpage of huge page to split. Split doesn't change the position of @page.
3435 *
3436 * Only caller must hold pin on the @page, otherwise split fails with -EBUSY.
3437 * The huge page must be locked.
3438 *
3439 * If @list is null, tail pages will be added to LRU list, otherwise, to @list.
3440 *
3441 * Both head page and tail pages will inherit mapping, flags, and so on from
3442 * the hugepage.
3443 *
3444 * GUP pin and PG_locked transferred to @page. Rest subpages can be freed if
3445 * they are not mapped.
3446 *
3447 * Returns 0 if the hugepage is split successfully.
3448 * Returns -EBUSY if the page is pinned or if anon_vma disappeared from under
3449 * us.
3450 */
3451int split_huge_page_to_list(struct page *page, struct list_head *list)
3452{
3453 struct page *head = compound_head(page);
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003454 struct pglist_data *pgdata = NODE_DATA(page_to_nid(head));
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003455 struct anon_vma *anon_vma = NULL;
3456 struct address_space *mapping = NULL;
3457 int count, mapcount, extra_pins, ret;
Kirill A. Shutemovd9654322016-01-15 16:54:43 -08003458 bool mlocked;
Kirill A. Shutemov0b9b6ff2016-01-20 14:58:09 -08003459 unsigned long flags;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003460
3461 VM_BUG_ON_PAGE(is_huge_zero_page(page), page);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003462 VM_BUG_ON_PAGE(!PageLocked(page), page);
3463 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
3464 VM_BUG_ON_PAGE(!PageCompound(page), page);
3465
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003466 if (PageAnon(head)) {
3467 /*
3468 * The caller does not necessarily hold an mmap_sem that would
3469 * prevent the anon_vma disappearing so we first we take a
3470 * reference to it and then lock the anon_vma for write. This
3471 * is similar to page_lock_anon_vma_read except the write lock
3472 * is taken to serialise against parallel split or collapse
3473 * operations.
3474 */
3475 anon_vma = page_get_anon_vma(head);
3476 if (!anon_vma) {
3477 ret = -EBUSY;
3478 goto out;
3479 }
3480 extra_pins = 0;
3481 mapping = NULL;
3482 anon_vma_lock_write(anon_vma);
3483 } else {
3484 mapping = head->mapping;
3485
3486 /* Truncated ? */
3487 if (!mapping) {
3488 ret = -EBUSY;
3489 goto out;
3490 }
3491
3492 /* Addidional pins from radix tree */
3493 extra_pins = HPAGE_PMD_NR;
3494 anon_vma = NULL;
3495 i_mmap_lock_read(mapping);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003496 }
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003497
3498 /*
3499 * Racy check if we can split the page, before freeze_page() will
3500 * split PMDs
3501 */
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003502 if (total_mapcount(head) != page_count(head) - extra_pins - 1) {
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003503 ret = -EBUSY;
3504 goto out_unlock;
3505 }
3506
Kirill A. Shutemovd9654322016-01-15 16:54:43 -08003507 mlocked = PageMlocked(page);
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003508 freeze_page(head);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003509 VM_BUG_ON_PAGE(compound_mapcount(head), head);
3510
Kirill A. Shutemovd9654322016-01-15 16:54:43 -08003511 /* Make sure the page is not on per-CPU pagevec as it takes pin */
3512 if (mlocked)
3513 lru_add_drain();
3514
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003515 /* prevent PageLRU to go away from under us, and freeze lru stats */
3516 spin_lock_irqsave(&page_zone(head)->lru_lock, flags);
3517
3518 if (mapping) {
3519 void **pslot;
3520
3521 spin_lock(&mapping->tree_lock);
3522 pslot = radix_tree_lookup_slot(&mapping->page_tree,
3523 page_index(head));
3524 /*
3525 * Check if the head page is present in radix tree.
3526 * We assume all tail are present too, if head is there.
3527 */
3528 if (radix_tree_deref_slot_protected(pslot,
3529 &mapping->tree_lock) != head)
3530 goto fail;
3531 }
3532
Joonsoo Kim0139aa72016-05-19 17:10:49 -07003533 /* Prevent deferred_split_scan() touching ->_refcount */
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003534 spin_lock(&pgdata->split_queue_lock);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003535 count = page_count(head);
3536 mapcount = total_mapcount(head);
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003537 if (!mapcount && page_ref_freeze(head, 1 + extra_pins)) {
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003538 if (!list_empty(page_deferred_list(head))) {
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003539 pgdata->split_queue_len--;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003540 list_del(page_deferred_list(head));
3541 }
Kirill A. Shutemov65c45372016-07-26 15:26:10 -07003542 if (mapping)
3543 __dec_zone_page_state(page, NR_SHMEM_THPS);
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003544 spin_unlock(&pgdata->split_queue_lock);
3545 __split_huge_page(page, list, flags);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003546 ret = 0;
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003547 } else {
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003548 if (IS_ENABLED(CONFIG_DEBUG_VM) && mapcount) {
3549 pr_alert("total_mapcount: %u, page_count(): %u\n",
3550 mapcount, count);
3551 if (PageTail(page))
3552 dump_page(head, NULL);
3553 dump_page(page, "total_mapcount(head) > 0");
3554 BUG();
3555 }
3556 spin_unlock(&pgdata->split_queue_lock);
3557fail: if (mapping)
3558 spin_unlock(&mapping->tree_lock);
3559 spin_unlock_irqrestore(&page_zone(head)->lru_lock, flags);
Kirill A. Shutemovfec89c12016-03-17 14:20:10 -07003560 unfreeze_page(head);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003561 ret = -EBUSY;
3562 }
3563
3564out_unlock:
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003565 if (anon_vma) {
3566 anon_vma_unlock_write(anon_vma);
3567 put_anon_vma(anon_vma);
3568 }
3569 if (mapping)
3570 i_mmap_unlock_read(mapping);
Kirill A. Shutemove9b61f12016-01-15 16:54:10 -08003571out:
3572 count_vm_event(!ret ? THP_SPLIT_PAGE : THP_SPLIT_PAGE_FAILED);
3573 return ret;
3574}
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003575
3576void free_transhuge_page(struct page *page)
3577{
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003578 struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003579 unsigned long flags;
3580
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003581 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003582 if (!list_empty(page_deferred_list(page))) {
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003583 pgdata->split_queue_len--;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003584 list_del(page_deferred_list(page));
3585 }
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003586 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003587 free_compound_page(page);
3588}
3589
3590void deferred_split_huge_page(struct page *page)
3591{
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003592 struct pglist_data *pgdata = NODE_DATA(page_to_nid(page));
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003593 unsigned long flags;
3594
3595 VM_BUG_ON_PAGE(!PageTransHuge(page), page);
3596
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003597 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003598 if (list_empty(page_deferred_list(page))) {
Kirill A. Shutemovf9719a02016-03-17 14:18:45 -07003599 count_vm_event(THP_DEFERRED_SPLIT_PAGE);
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003600 list_add_tail(page_deferred_list(page), &pgdata->split_queue);
3601 pgdata->split_queue_len++;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003602 }
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003603 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003604}
3605
3606static unsigned long deferred_split_count(struct shrinker *shrink,
3607 struct shrink_control *sc)
3608{
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003609 struct pglist_data *pgdata = NODE_DATA(sc->nid);
Kirill A. Shutemovcb8d68e2016-02-02 16:57:12 -08003610 return ACCESS_ONCE(pgdata->split_queue_len);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003611}
3612
3613static unsigned long deferred_split_scan(struct shrinker *shrink,
3614 struct shrink_control *sc)
3615{
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003616 struct pglist_data *pgdata = NODE_DATA(sc->nid);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003617 unsigned long flags;
3618 LIST_HEAD(list), *pos, *next;
3619 struct page *page;
3620 int split = 0;
3621
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003622 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003623 /* Take pin on all head pages to avoid freeing them under us */
Kirill A. Shutemovae026202016-02-05 15:36:53 -08003624 list_for_each_safe(pos, next, &pgdata->split_queue) {
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003625 page = list_entry((void *)pos, struct page, mapping);
3626 page = compound_head(page);
Kirill A. Shutemove3ae1952016-02-02 16:57:15 -08003627 if (get_page_unless_zero(page)) {
3628 list_move(page_deferred_list(page), &list);
3629 } else {
3630 /* We lost race with put_compound_page() */
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003631 list_del_init(page_deferred_list(page));
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003632 pgdata->split_queue_len--;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003633 }
Kirill A. Shutemove3ae1952016-02-02 16:57:15 -08003634 if (!--sc->nr_to_scan)
3635 break;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003636 }
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003637 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003638
3639 list_for_each_safe(pos, next, &list) {
3640 page = list_entry((void *)pos, struct page, mapping);
3641 lock_page(page);
3642 /* split_huge_page() removes page from list on success */
3643 if (!split_huge_page(page))
3644 split++;
3645 unlock_page(page);
3646 put_page(page);
3647 }
3648
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003649 spin_lock_irqsave(&pgdata->split_queue_lock, flags);
3650 list_splice_tail(&list, &pgdata->split_queue);
3651 spin_unlock_irqrestore(&pgdata->split_queue_lock, flags);
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003652
Kirill A. Shutemovcb8d68e2016-02-02 16:57:12 -08003653 /*
3654 * Stop shrinker if we didn't split any page, but the queue is empty.
3655 * This can happen if pages were freed under us.
3656 */
3657 if (!split && list_empty(&pgdata->split_queue))
3658 return SHRINK_STOP;
3659 return split;
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003660}
3661
3662static struct shrinker deferred_split_shrinker = {
3663 .count_objects = deferred_split_count,
3664 .scan_objects = deferred_split_scan,
3665 .seeks = DEFAULT_SEEKS,
Kirill A. Shutemova3d0a9182016-02-02 16:57:08 -08003666 .flags = SHRINKER_NUMA_AWARE,
Kirill A. Shutemov9a982252016-01-15 16:54:17 -08003667};
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08003668
3669#ifdef CONFIG_DEBUG_FS
3670static int split_huge_pages_set(void *data, u64 val)
3671{
3672 struct zone *zone;
3673 struct page *page;
3674 unsigned long pfn, max_zone_pfn;
3675 unsigned long total = 0, split = 0;
3676
3677 if (val != 1)
3678 return -EINVAL;
3679
3680 for_each_populated_zone(zone) {
3681 max_zone_pfn = zone_end_pfn(zone);
3682 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++) {
3683 if (!pfn_valid(pfn))
3684 continue;
3685
3686 page = pfn_to_page(pfn);
3687 if (!get_page_unless_zero(page))
3688 continue;
3689
3690 if (zone != page_zone(page))
3691 goto next;
3692
Kirill A. Shutemovbaa355f2016-07-26 15:25:51 -07003693 if (!PageHead(page) || PageHuge(page) || !PageLRU(page))
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08003694 goto next;
3695
3696 total++;
3697 lock_page(page);
3698 if (!split_huge_page(page))
3699 split++;
3700 unlock_page(page);
3701next:
3702 put_page(page);
3703 }
3704 }
3705
Yang Shi145bdaa2016-05-05 16:22:00 -07003706 pr_info("%lu of %lu THP split\n", split, total);
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08003707
3708 return 0;
3709}
3710DEFINE_SIMPLE_ATTRIBUTE(split_huge_pages_fops, NULL, split_huge_pages_set,
3711 "%llu\n");
3712
3713static int __init split_huge_pages_debugfs(void)
3714{
3715 void *ret;
3716
Yang Shi145bdaa2016-05-05 16:22:00 -07003717 ret = debugfs_create_file("split_huge_pages", 0200, NULL, NULL,
Kirill A. Shutemov49071d42016-01-15 16:54:40 -08003718 &split_huge_pages_fops);
3719 if (!ret)
3720 pr_warn("Failed to create split_huge_pages in debugfs");
3721 return 0;
3722}
3723late_initcall(split_huge_pages_debugfs);
3724#endif