Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 1 | #ifndef _LINUX_HUGE_MM_H |
| 2 | #define _LINUX_HUGE_MM_H |
| 3 | |
| 4 | extern int do_huge_pmd_anonymous_page(struct mm_struct *mm, |
| 5 | struct vm_area_struct *vma, |
| 6 | unsigned long address, pmd_t *pmd, |
| 7 | unsigned int flags); |
| 8 | extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, |
| 9 | pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, |
| 10 | struct vm_area_struct *vma); |
Will Deacon | a1dd450 | 2012-12-11 16:01:27 -0800 | [diff] [blame] | 11 | extern void huge_pmd_set_accessed(struct mm_struct *mm, |
| 12 | struct vm_area_struct *vma, |
| 13 | unsigned long address, pmd_t *pmd, |
| 14 | pmd_t orig_pmd, int dirty); |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 15 | extern int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, |
| 16 | unsigned long address, pmd_t *pmd, |
| 17 | pmd_t orig_pmd); |
David Rientjes | b676b29 | 2012-10-08 16:34:03 -0700 | [diff] [blame] | 18 | extern struct page *follow_trans_huge_pmd(struct vm_area_struct *vma, |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 19 | unsigned long addr, |
| 20 | pmd_t *pmd, |
| 21 | unsigned int flags); |
| 22 | extern int zap_huge_pmd(struct mmu_gather *tlb, |
| 23 | struct vm_area_struct *vma, |
Shaohua Li | f21760b | 2012-01-12 17:19:16 -0800 | [diff] [blame] | 24 | pmd_t *pmd, unsigned long addr); |
Johannes Weiner | 0ca1634 | 2011-01-13 15:47:02 -0800 | [diff] [blame] | 25 | extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, |
| 26 | unsigned long addr, unsigned long end, |
| 27 | unsigned char *vec); |
Andrea Arcangeli | 37a1c49 | 2011-10-31 17:08:30 -0700 | [diff] [blame] | 28 | extern int move_huge_pmd(struct vm_area_struct *vma, |
| 29 | struct vm_area_struct *new_vma, |
| 30 | unsigned long old_addr, |
| 31 | unsigned long new_addr, unsigned long old_end, |
| 32 | pmd_t *old_pmd, pmd_t *new_pmd); |
Johannes Weiner | cd7548a | 2011-01-13 15:47:04 -0800 | [diff] [blame] | 33 | extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, |
Mel Gorman | 4b10e7d | 2012-10-25 14:16:32 +0200 | [diff] [blame] | 34 | unsigned long addr, pgprot_t newprot, |
| 35 | int prot_numa); |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 36 | |
| 37 | enum transparent_hugepage_flag { |
| 38 | TRANSPARENT_HUGEPAGE_FLAG, |
| 39 | TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, |
| 40 | TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, |
| 41 | TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, |
Andrea Arcangeli | ba76149 | 2011-01-13 15:46:58 -0800 | [diff] [blame] | 42 | TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG, |
Kirill A. Shutemov | 79da540 | 2012-12-12 13:51:12 -0800 | [diff] [blame] | 43 | TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG, |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 44 | #ifdef CONFIG_DEBUG_VM |
| 45 | TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG, |
| 46 | #endif |
| 47 | }; |
| 48 | |
| 49 | enum page_check_address_pmd_flag { |
| 50 | PAGE_CHECK_ADDRESS_PMD_FLAG, |
| 51 | PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG, |
| 52 | PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG, |
| 53 | }; |
| 54 | extern pmd_t *page_check_address_pmd(struct page *page, |
| 55 | struct mm_struct *mm, |
| 56 | unsigned long address, |
Kirill A. Shutemov | 117b079 | 2013-11-14 14:30:56 -0800 | [diff] [blame] | 57 | enum page_check_address_pmd_flag flag, |
| 58 | spinlock_t **ptl); |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 59 | |
Naoya Horiguchi | d8c37c4 | 2012-03-21 16:34:27 -0700 | [diff] [blame] | 60 | #define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT) |
| 61 | #define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER) |
| 62 | |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 63 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
Aneesh Kumar K.V | fde5279 | 2013-06-05 17:14:05 -0700 | [diff] [blame] | 64 | #define HPAGE_PMD_SHIFT PMD_SHIFT |
| 65 | #define HPAGE_PMD_SIZE ((1UL) << HPAGE_PMD_SHIFT) |
| 66 | #define HPAGE_PMD_MASK (~(HPAGE_PMD_SIZE - 1)) |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 67 | |
Alex Shi | 2099597 | 2012-05-29 15:06:31 -0700 | [diff] [blame] | 68 | extern bool is_vma_temporary_stack(struct vm_area_struct *vma); |
| 69 | |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 70 | #define transparent_hugepage_enabled(__vma) \ |
Andrea Arcangeli | a664b2d | 2011-01-13 15:47:17 -0800 | [diff] [blame] | 71 | ((transparent_hugepage_flags & \ |
| 72 | (1<<TRANSPARENT_HUGEPAGE_FLAG) || \ |
| 73 | (transparent_hugepage_flags & \ |
| 74 | (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG) && \ |
| 75 | ((__vma)->vm_flags & VM_HUGEPAGE))) && \ |
Andrea Arcangeli | a7d6e4e | 2011-02-15 19:02:45 +0100 | [diff] [blame] | 76 | !((__vma)->vm_flags & VM_NOHUGEPAGE) && \ |
| 77 | !is_vma_temporary_stack(__vma)) |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 78 | #define transparent_hugepage_defrag(__vma) \ |
| 79 | ((transparent_hugepage_flags & \ |
| 80 | (1<<TRANSPARENT_HUGEPAGE_DEFRAG_FLAG)) || \ |
| 81 | (transparent_hugepage_flags & \ |
| 82 | (1<<TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG) && \ |
| 83 | (__vma)->vm_flags & VM_HUGEPAGE)) |
Kirill A. Shutemov | 79da540 | 2012-12-12 13:51:12 -0800 | [diff] [blame] | 84 | #define transparent_hugepage_use_zero_page() \ |
| 85 | (transparent_hugepage_flags & \ |
| 86 | (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG)) |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 87 | #ifdef CONFIG_DEBUG_VM |
| 88 | #define transparent_hugepage_debug_cow() \ |
| 89 | (transparent_hugepage_flags & \ |
| 90 | (1<<TRANSPARENT_HUGEPAGE_DEBUG_COW_FLAG)) |
| 91 | #else /* CONFIG_DEBUG_VM */ |
| 92 | #define transparent_hugepage_debug_cow() 0 |
| 93 | #endif /* CONFIG_DEBUG_VM */ |
| 94 | |
| 95 | extern unsigned long transparent_hugepage_flags; |
| 96 | extern int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, |
| 97 | pmd_t *dst_pmd, pmd_t *src_pmd, |
| 98 | struct vm_area_struct *vma, |
| 99 | unsigned long addr, unsigned long end); |
Shaohua Li | 5bc7b8a | 2013-04-29 15:08:36 -0700 | [diff] [blame] | 100 | extern int split_huge_page_to_list(struct page *page, struct list_head *list); |
| 101 | static inline int split_huge_page(struct page *page) |
| 102 | { |
| 103 | return split_huge_page_to_list(page, NULL); |
| 104 | } |
Kirill A. Shutemov | e180377 | 2012-12-12 13:50:59 -0800 | [diff] [blame] | 105 | extern void __split_huge_page_pmd(struct vm_area_struct *vma, |
| 106 | unsigned long address, pmd_t *pmd); |
| 107 | #define split_huge_page_pmd(__vma, __address, __pmd) \ |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 108 | do { \ |
| 109 | pmd_t *____pmd = (__pmd); \ |
| 110 | if (unlikely(pmd_trans_huge(*____pmd))) \ |
Kirill A. Shutemov | e180377 | 2012-12-12 13:50:59 -0800 | [diff] [blame] | 111 | __split_huge_page_pmd(__vma, __address, \ |
| 112 | ____pmd); \ |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 113 | } while (0) |
| 114 | #define wait_split_huge_page(__anon_vma, __pmd) \ |
| 115 | do { \ |
| 116 | pmd_t *____pmd = (__pmd); \ |
Ingo Molnar | 4fc3f1d | 2012-12-02 19:56:50 +0000 | [diff] [blame] | 117 | anon_vma_lock_write(__anon_vma); \ |
Konstantin Khlebnikov | 08b5270 | 2013-02-22 16:34:40 -0800 | [diff] [blame] | 118 | anon_vma_unlock_write(__anon_vma); \ |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 119 | BUG_ON(pmd_trans_splitting(*____pmd) || \ |
| 120 | pmd_trans_huge(*____pmd)); \ |
| 121 | } while (0) |
Kirill A. Shutemov | e180377 | 2012-12-12 13:50:59 -0800 | [diff] [blame] | 122 | extern void split_huge_page_pmd_mm(struct mm_struct *mm, unsigned long address, |
| 123 | pmd_t *pmd); |
Steve Capper | dfa5e23 | 2013-05-07 14:46:03 +0100 | [diff] [blame] | 124 | #if HPAGE_PMD_ORDER >= MAX_ORDER |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 125 | #error "hugepages can't be allocated by the buddy allocator" |
| 126 | #endif |
Andrea Arcangeli | 60ab324 | 2011-01-13 15:47:18 -0800 | [diff] [blame] | 127 | extern int hugepage_madvise(struct vm_area_struct *vma, |
| 128 | unsigned long *vm_flags, int advice); |
Andrea Arcangeli | 94fcc58 | 2011-01-13 15:47:08 -0800 | [diff] [blame] | 129 | extern void __vma_adjust_trans_huge(struct vm_area_struct *vma, |
| 130 | unsigned long start, |
| 131 | unsigned long end, |
| 132 | long adjust_next); |
Kirill A. Shutemov | bf92915 | 2013-11-14 14:30:54 -0800 | [diff] [blame] | 133 | extern int __pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma, |
| 134 | spinlock_t **ptl); |
Naoya Horiguchi | 025c5b2 | 2012-03-21 16:33:57 -0700 | [diff] [blame] | 135 | /* mmap_sem must be held on entry */ |
Kirill A. Shutemov | bf92915 | 2013-11-14 14:30:54 -0800 | [diff] [blame] | 136 | static inline int pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma, |
| 137 | spinlock_t **ptl) |
Naoya Horiguchi | 025c5b2 | 2012-03-21 16:33:57 -0700 | [diff] [blame] | 138 | { |
| 139 | VM_BUG_ON(!rwsem_is_locked(&vma->vm_mm->mmap_sem)); |
| 140 | if (pmd_trans_huge(*pmd)) |
Kirill A. Shutemov | bf92915 | 2013-11-14 14:30:54 -0800 | [diff] [blame] | 141 | return __pmd_trans_huge_lock(pmd, vma, ptl); |
Naoya Horiguchi | 025c5b2 | 2012-03-21 16:33:57 -0700 | [diff] [blame] | 142 | else |
| 143 | return 0; |
| 144 | } |
Andrea Arcangeli | 94fcc58 | 2011-01-13 15:47:08 -0800 | [diff] [blame] | 145 | static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, |
| 146 | unsigned long start, |
| 147 | unsigned long end, |
| 148 | long adjust_next) |
| 149 | { |
Andrea Arcangeli | 78f11a2 | 2011-04-27 15:26:45 -0700 | [diff] [blame] | 150 | if (!vma->anon_vma || vma->vm_ops) |
Andrea Arcangeli | 94fcc58 | 2011-01-13 15:47:08 -0800 | [diff] [blame] | 151 | return; |
| 152 | __vma_adjust_trans_huge(vma, start, end, adjust_next); |
| 153 | } |
Rik van Riel | 2c888cf | 2011-01-13 15:47:13 -0800 | [diff] [blame] | 154 | static inline int hpage_nr_pages(struct page *page) |
| 155 | { |
| 156 | if (unlikely(PageTransHuge(page))) |
| 157 | return HPAGE_PMD_NR; |
| 158 | return 1; |
| 159 | } |
Mel Gorman | d10e63f | 2012-10-25 14:16:31 +0200 | [diff] [blame] | 160 | |
Mel Gorman | 4daae3b | 2012-11-02 11:33:45 +0000 | [diff] [blame] | 161 | extern int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, |
| 162 | unsigned long addr, pmd_t pmd, pmd_t *pmdp); |
Mel Gorman | d10e63f | 2012-10-25 14:16:31 +0200 | [diff] [blame] | 163 | |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 164 | #else /* CONFIG_TRANSPARENT_HUGEPAGE */ |
Naoya Horiguchi | d8c37c4 | 2012-03-21 16:34:27 -0700 | [diff] [blame] | 165 | #define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; }) |
| 166 | #define HPAGE_PMD_MASK ({ BUILD_BUG(); 0; }) |
| 167 | #define HPAGE_PMD_SIZE ({ BUILD_BUG(); 0; }) |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 168 | |
Rik van Riel | 2c888cf | 2011-01-13 15:47:13 -0800 | [diff] [blame] | 169 | #define hpage_nr_pages(x) 1 |
| 170 | |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 171 | #define transparent_hugepage_enabled(__vma) 0 |
| 172 | |
| 173 | #define transparent_hugepage_flags 0UL |
Shaohua Li | 5bc7b8a | 2013-04-29 15:08:36 -0700 | [diff] [blame] | 174 | static inline int |
| 175 | split_huge_page_to_list(struct page *page, struct list_head *list) |
| 176 | { |
| 177 | return 0; |
| 178 | } |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 179 | static inline int split_huge_page(struct page *page) |
| 180 | { |
| 181 | return 0; |
| 182 | } |
Kirill A. Shutemov | e180377 | 2012-12-12 13:50:59 -0800 | [diff] [blame] | 183 | #define split_huge_page_pmd(__vma, __address, __pmd) \ |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 184 | do { } while (0) |
| 185 | #define wait_split_huge_page(__anon_vma, __pmd) \ |
| 186 | do { } while (0) |
Kirill A. Shutemov | e180377 | 2012-12-12 13:50:59 -0800 | [diff] [blame] | 187 | #define split_huge_page_pmd_mm(__mm, __address, __pmd) \ |
| 188 | do { } while (0) |
Andrea Arcangeli | 60ab324 | 2011-01-13 15:47:18 -0800 | [diff] [blame] | 189 | static inline int hugepage_madvise(struct vm_area_struct *vma, |
| 190 | unsigned long *vm_flags, int advice) |
Andrea Arcangeli | 0af4e98 | 2011-01-13 15:46:55 -0800 | [diff] [blame] | 191 | { |
| 192 | BUG(); |
| 193 | return 0; |
| 194 | } |
Andrea Arcangeli | 94fcc58 | 2011-01-13 15:47:08 -0800 | [diff] [blame] | 195 | static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, |
| 196 | unsigned long start, |
| 197 | unsigned long end, |
| 198 | long adjust_next) |
| 199 | { |
| 200 | } |
Kirill A. Shutemov | bf92915 | 2013-11-14 14:30:54 -0800 | [diff] [blame] | 201 | static inline int pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma, |
| 202 | spinlock_t **ptl) |
Naoya Horiguchi | 025c5b2 | 2012-03-21 16:33:57 -0700 | [diff] [blame] | 203 | { |
| 204 | return 0; |
| 205 | } |
Mel Gorman | d10e63f | 2012-10-25 14:16:31 +0200 | [diff] [blame] | 206 | |
Mel Gorman | 4daae3b | 2012-11-02 11:33:45 +0000 | [diff] [blame] | 207 | static inline int do_huge_pmd_numa_page(struct mm_struct *mm, struct vm_area_struct *vma, |
| 208 | unsigned long addr, pmd_t pmd, pmd_t *pmdp) |
Mel Gorman | d10e63f | 2012-10-25 14:16:31 +0200 | [diff] [blame] | 209 | { |
Mel Gorman | 4daae3b | 2012-11-02 11:33:45 +0000 | [diff] [blame] | 210 | return 0; |
Mel Gorman | d10e63f | 2012-10-25 14:16:31 +0200 | [diff] [blame] | 211 | } |
| 212 | |
Andrea Arcangeli | 71e3aac | 2011-01-13 15:46:52 -0800 | [diff] [blame] | 213 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
| 214 | |
| 215 | #endif /* _LINUX_HUGE_MM_H */ |