Greg Kroah-Hartman | b244131 | 2017-11-01 15:07:57 +0100 | [diff] [blame] | 1 | // SPDX-License-Identifier: GPL-2.0 |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 2 | #include <linux/debugfs.h> |
| 3 | #include <linux/mm.h> |
| 4 | #include <linux/slab.h> |
| 5 | #include <linux/uaccess.h> |
| 6 | #include <linux/bootmem.h> |
| 7 | #include <linux/stacktrace.h> |
| 8 | #include <linux/page_owner.h> |
Vlastimil Babka | 7dd80b8 | 2016-03-15 14:56:12 -0700 | [diff] [blame] | 9 | #include <linux/jump_label.h> |
Vlastimil Babka | 7cd12b4 | 2016-03-15 14:56:18 -0700 | [diff] [blame] | 10 | #include <linux/migrate.h> |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 11 | #include <linux/stackdepot.h> |
Joonsoo Kim | e2f612e | 2016-10-07 16:58:21 -0700 | [diff] [blame] | 12 | #include <linux/seq_file.h> |
Liam Mark | ea0195f | 2017-12-21 14:27:20 -0800 | [diff] [blame] | 13 | #include <linux/sched.h> |
| 14 | #include <linux/sched/clock.h> |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 15 | |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 16 | #include "internal.h" |
| 17 | |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 18 | /* |
| 19 | * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack) |
| 20 | * to use off stack temporal storage |
| 21 | */ |
| 22 | #define PAGE_OWNER_STACK_DEPTH (16) |
| 23 | |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 24 | struct page_owner { |
Ayush Mittal | 6b4c54e | 2017-11-15 17:34:30 -0800 | [diff] [blame] | 25 | unsigned short order; |
| 26 | short last_migrate_reason; |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 27 | gfp_t gfp_mask; |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 28 | depot_stack_handle_t handle; |
Liam Mark | ea0195f | 2017-12-21 14:27:20 -0800 | [diff] [blame] | 29 | int pid; |
| 30 | u64 ts_nsec; |
Sudarshan Rajagopalan | 0b243ad | 2019-10-09 15:01:23 -0700 | [diff] [blame] | 31 | u64 free_ts_nsec; |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 32 | }; |
| 33 | |
Trilok Soni | 6fe7396 | 2016-04-20 18:12:51 -0700 | [diff] [blame] | 34 | static bool page_owner_disabled = |
| 35 | !IS_ENABLED(CONFIG_PAGE_OWNER_ENABLE_DEFAULT); |
Vlastimil Babka | 7dd80b8 | 2016-03-15 14:56:12 -0700 | [diff] [blame] | 36 | DEFINE_STATIC_KEY_FALSE(page_owner_inited); |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 37 | |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 38 | static depot_stack_handle_t dummy_handle; |
| 39 | static depot_stack_handle_t failure_handle; |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 40 | static depot_stack_handle_t early_handle; |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 41 | |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 42 | static void init_early_allocated_pages(void); |
| 43 | |
Dou Liyang | 1173194 | 2018-04-05 16:23:49 -0700 | [diff] [blame] | 44 | static int __init early_page_owner_param(char *buf) |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 45 | { |
| 46 | if (!buf) |
| 47 | return -EINVAL; |
| 48 | |
| 49 | if (strcmp(buf, "on") == 0) |
| 50 | page_owner_disabled = false; |
| 51 | |
Trilok Soni | 6fe7396 | 2016-04-20 18:12:51 -0700 | [diff] [blame] | 52 | if (strcmp(buf, "off") == 0) |
| 53 | page_owner_disabled = true; |
| 54 | |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 55 | return 0; |
| 56 | } |
| 57 | early_param("page_owner", early_page_owner_param); |
| 58 | |
| 59 | static bool need_page_owner(void) |
| 60 | { |
| 61 | if (page_owner_disabled) |
| 62 | return false; |
| 63 | |
| 64 | return true; |
| 65 | } |
| 66 | |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 67 | static __always_inline depot_stack_handle_t create_dummy_stack(void) |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 68 | { |
| 69 | unsigned long entries[4]; |
| 70 | struct stack_trace dummy; |
| 71 | |
| 72 | dummy.nr_entries = 0; |
| 73 | dummy.max_entries = ARRAY_SIZE(entries); |
| 74 | dummy.entries = &entries[0]; |
| 75 | dummy.skip = 0; |
| 76 | |
| 77 | save_stack_trace(&dummy); |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 78 | return depot_save_stack(&dummy, GFP_KERNEL); |
| 79 | } |
| 80 | |
| 81 | static noinline void register_dummy_stack(void) |
| 82 | { |
| 83 | dummy_handle = create_dummy_stack(); |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 84 | } |
| 85 | |
| 86 | static noinline void register_failure_stack(void) |
| 87 | { |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 88 | failure_handle = create_dummy_stack(); |
| 89 | } |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 90 | |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 91 | static noinline void register_early_stack(void) |
| 92 | { |
| 93 | early_handle = create_dummy_stack(); |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 94 | } |
| 95 | |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 96 | static void init_page_owner(void) |
| 97 | { |
| 98 | if (page_owner_disabled) |
| 99 | return; |
| 100 | |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 101 | register_dummy_stack(); |
| 102 | register_failure_stack(); |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 103 | register_early_stack(); |
Vlastimil Babka | 7dd80b8 | 2016-03-15 14:56:12 -0700 | [diff] [blame] | 104 | static_branch_enable(&page_owner_inited); |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 105 | init_early_allocated_pages(); |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 106 | } |
| 107 | |
| 108 | struct page_ext_operations page_owner_ops = { |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 109 | .size = sizeof(struct page_owner), |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 110 | .need = need_page_owner, |
| 111 | .init = init_page_owner, |
| 112 | }; |
| 113 | |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 114 | static inline struct page_owner *get_page_owner(struct page_ext *page_ext) |
| 115 | { |
| 116 | return (void *)page_ext + page_owner_ops.offset; |
| 117 | } |
| 118 | |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 119 | void __reset_page_owner(struct page *page, unsigned int order) |
| 120 | { |
| 121 | int i; |
| 122 | struct page_ext *page_ext; |
Sudarshan Rajagopalan | 0b243ad | 2019-10-09 15:01:23 -0700 | [diff] [blame] | 123 | u64 free_ts_nsec = local_clock(); |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 124 | |
| 125 | for (i = 0; i < (1 << order); i++) { |
| 126 | page_ext = lookup_page_ext(page + i); |
Yang Shi | f86e427 | 2016-06-03 14:55:38 -0700 | [diff] [blame] | 127 | if (unlikely(!page_ext)) |
| 128 | continue; |
Sudarshan Rajagopalan | 0b243ad | 2019-10-09 15:01:23 -0700 | [diff] [blame] | 129 | get_page_owner(page_ext)->free_ts_nsec = free_ts_nsec; |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 130 | __clear_bit(PAGE_EXT_OWNER, &page_ext->flags); |
Sudarshan Rajagopalan | 0b243ad | 2019-10-09 15:01:23 -0700 | [diff] [blame] | 131 | __set_bit(PAGE_EXT_PG_FREE, &page_ext->flags); |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 132 | } |
| 133 | } |
| 134 | |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 135 | static inline bool check_recursive_alloc(struct stack_trace *trace, |
| 136 | unsigned long ip) |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 137 | { |
Maninder Singh | 299815a | 2018-03-28 16:01:05 -0700 | [diff] [blame] | 138 | int i; |
Yang Shi | f86e427 | 2016-06-03 14:55:38 -0700 | [diff] [blame] | 139 | |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 140 | if (!trace->nr_entries) |
| 141 | return false; |
| 142 | |
Maninder Singh | 299815a | 2018-03-28 16:01:05 -0700 | [diff] [blame] | 143 | for (i = 0; i < trace->nr_entries; i++) { |
| 144 | if (trace->entries[i] == ip) |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 145 | return true; |
| 146 | } |
| 147 | |
| 148 | return false; |
| 149 | } |
| 150 | |
| 151 | static noinline depot_stack_handle_t save_stack(gfp_t flags) |
| 152 | { |
| 153 | unsigned long entries[PAGE_OWNER_STACK_DEPTH]; |
Sergei Rogachev | 94f759d6 | 2015-02-11 15:28:34 -0800 | [diff] [blame] | 154 | struct stack_trace trace = { |
| 155 | .nr_entries = 0, |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 156 | .entries = entries, |
| 157 | .max_entries = PAGE_OWNER_STACK_DEPTH, |
Prakash Gupta | 5f48f0b | 2017-09-13 16:28:35 -0700 | [diff] [blame] | 158 | .skip = 2 |
Sergei Rogachev | 94f759d6 | 2015-02-11 15:28:34 -0800 | [diff] [blame] | 159 | }; |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 160 | depot_stack_handle_t handle; |
| 161 | |
| 162 | save_stack_trace(&trace); |
| 163 | if (trace.nr_entries != 0 && |
| 164 | trace.entries[trace.nr_entries-1] == ULONG_MAX) |
| 165 | trace.nr_entries--; |
| 166 | |
| 167 | /* |
| 168 | * We need to check recursion here because our request to stackdepot |
| 169 | * could trigger memory allocation to save new entry. New memory |
| 170 | * allocation would reach here and call depot_save_stack() again |
| 171 | * if we don't catch it. There is still not enough memory in stackdepot |
| 172 | * so it would try to allocate memory again and loop forever. |
| 173 | */ |
| 174 | if (check_recursive_alloc(&trace, _RET_IP_)) |
| 175 | return dummy_handle; |
| 176 | |
| 177 | handle = depot_save_stack(&trace, flags); |
| 178 | if (!handle) |
| 179 | handle = failure_handle; |
| 180 | |
| 181 | return handle; |
| 182 | } |
| 183 | |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 184 | static inline void __set_page_owner_handle(struct page_ext *page_ext, |
| 185 | depot_stack_handle_t handle, unsigned int order, gfp_t gfp_mask) |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 186 | { |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 187 | struct page_owner *page_owner; |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 188 | |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 189 | page_owner = get_page_owner(page_ext); |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 190 | page_owner->handle = handle; |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 191 | page_owner->order = order; |
| 192 | page_owner->gfp_mask = gfp_mask; |
| 193 | page_owner->last_migrate_reason = -1; |
Liam Mark | ea0195f | 2017-12-21 14:27:20 -0800 | [diff] [blame] | 194 | page_owner->pid = current->pid; |
| 195 | page_owner->ts_nsec = local_clock(); |
Sudarshan Rajagopalan | 0b243ad | 2019-10-09 15:01:23 -0700 | [diff] [blame] | 196 | page_owner->free_ts_nsec = 0; |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 197 | |
| 198 | __set_bit(PAGE_EXT_OWNER, &page_ext->flags); |
Sudarshan Rajagopalan | 0b243ad | 2019-10-09 15:01:23 -0700 | [diff] [blame] | 199 | __clear_bit(PAGE_EXT_PG_FREE, &page_ext->flags); |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 200 | } |
| 201 | |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 202 | noinline void __set_page_owner(struct page *page, unsigned int order, |
| 203 | gfp_t gfp_mask) |
| 204 | { |
| 205 | struct page_ext *page_ext = lookup_page_ext(page); |
| 206 | depot_stack_handle_t handle; |
Sudarshan Rajagopalan | 0b243ad | 2019-10-09 15:01:23 -0700 | [diff] [blame] | 207 | int i; |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 208 | |
| 209 | if (unlikely(!page_ext)) |
| 210 | return; |
| 211 | |
| 212 | handle = save_stack(gfp_mask); |
| 213 | __set_page_owner_handle(page_ext, handle, order, gfp_mask); |
Sudarshan Rajagopalan | 0b243ad | 2019-10-09 15:01:23 -0700 | [diff] [blame] | 214 | |
| 215 | /* set page owner for tail pages if any */ |
| 216 | for (i = 1; i < (1 << order); i++) { |
| 217 | page_ext = lookup_page_ext(page + i); |
| 218 | |
| 219 | if (unlikely(!page_ext)) |
| 220 | continue; |
| 221 | |
| 222 | /* mark tail pages as order 0 individual pages */ |
| 223 | __set_page_owner_handle(page_ext, handle, 0, gfp_mask); |
| 224 | } |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 225 | } |
| 226 | |
Vlastimil Babka | 7cd12b4 | 2016-03-15 14:56:18 -0700 | [diff] [blame] | 227 | void __set_page_owner_migrate_reason(struct page *page, int reason) |
| 228 | { |
| 229 | struct page_ext *page_ext = lookup_page_ext(page); |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 230 | struct page_owner *page_owner; |
| 231 | |
Yang Shi | f86e427 | 2016-06-03 14:55:38 -0700 | [diff] [blame] | 232 | if (unlikely(!page_ext)) |
| 233 | return; |
Vlastimil Babka | 7cd12b4 | 2016-03-15 14:56:18 -0700 | [diff] [blame] | 234 | |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 235 | page_owner = get_page_owner(page_ext); |
| 236 | page_owner->last_migrate_reason = reason; |
Vlastimil Babka | 7cd12b4 | 2016-03-15 14:56:18 -0700 | [diff] [blame] | 237 | } |
| 238 | |
Joonsoo Kim | a9627bc | 2016-07-26 15:23:49 -0700 | [diff] [blame] | 239 | void __split_page_owner(struct page *page, unsigned int order) |
Joonsoo Kim | e2cfc91 | 2015-07-17 16:24:18 -0700 | [diff] [blame] | 240 | { |
Joonsoo Kim | a9627bc | 2016-07-26 15:23:49 -0700 | [diff] [blame] | 241 | int i; |
Joonsoo Kim | e2cfc91 | 2015-07-17 16:24:18 -0700 | [diff] [blame] | 242 | struct page_ext *page_ext = lookup_page_ext(page); |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 243 | struct page_owner *page_owner; |
Joonsoo Kim | e2cfc91 | 2015-07-17 16:24:18 -0700 | [diff] [blame] | 244 | |
Joonsoo Kim | a9627bc | 2016-07-26 15:23:49 -0700 | [diff] [blame] | 245 | if (unlikely(!page_ext)) |
| 246 | return; |
| 247 | |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 248 | page_owner = get_page_owner(page_ext); |
| 249 | page_owner->order = 0; |
Joonsoo Kim | a9627bc | 2016-07-26 15:23:49 -0700 | [diff] [blame] | 250 | for (i = 1; i < (1 << order); i++) |
| 251 | __copy_page_owner(page, page + i); |
Joonsoo Kim | e2cfc91 | 2015-07-17 16:24:18 -0700 | [diff] [blame] | 252 | } |
| 253 | |
Vlastimil Babka | d435edc | 2016-03-15 14:56:15 -0700 | [diff] [blame] | 254 | void __copy_page_owner(struct page *oldpage, struct page *newpage) |
| 255 | { |
| 256 | struct page_ext *old_ext = lookup_page_ext(oldpage); |
| 257 | struct page_ext *new_ext = lookup_page_ext(newpage); |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 258 | struct page_owner *old_page_owner, *new_page_owner; |
Vlastimil Babka | d435edc | 2016-03-15 14:56:15 -0700 | [diff] [blame] | 259 | |
Yang Shi | f86e427 | 2016-06-03 14:55:38 -0700 | [diff] [blame] | 260 | if (unlikely(!old_ext || !new_ext)) |
| 261 | return; |
| 262 | |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 263 | old_page_owner = get_page_owner(old_ext); |
| 264 | new_page_owner = get_page_owner(new_ext); |
| 265 | new_page_owner->order = old_page_owner->order; |
| 266 | new_page_owner->gfp_mask = old_page_owner->gfp_mask; |
| 267 | new_page_owner->last_migrate_reason = |
| 268 | old_page_owner->last_migrate_reason; |
| 269 | new_page_owner->handle = old_page_owner->handle; |
Liam Mark | ea0195f | 2017-12-21 14:27:20 -0800 | [diff] [blame] | 270 | new_page_owner->pid = old_page_owner->pid; |
| 271 | new_page_owner->ts_nsec = old_page_owner->ts_nsec; |
Sudarshan Rajagopalan | 0b243ad | 2019-10-09 15:01:23 -0700 | [diff] [blame] | 272 | new_page_owner->free_ts_nsec = old_page_owner->ts_nsec; |
Vlastimil Babka | d435edc | 2016-03-15 14:56:15 -0700 | [diff] [blame] | 273 | |
| 274 | /* |
| 275 | * We don't clear the bit on the oldpage as it's going to be freed |
| 276 | * after migration. Until then, the info can be useful in case of |
| 277 | * a bug, and the overal stats will be off a bit only temporarily. |
| 278 | * Also, migrate_misplaced_transhuge_page() can still fail the |
| 279 | * migration and then we want the oldpage to retain the info. But |
| 280 | * in that case we also don't need to explicitly clear the info from |
| 281 | * the new page, which will be freed. |
| 282 | */ |
| 283 | __set_bit(PAGE_EXT_OWNER, &new_ext->flags); |
| 284 | } |
| 285 | |
Joonsoo Kim | e2f612e | 2016-10-07 16:58:21 -0700 | [diff] [blame] | 286 | void pagetypeinfo_showmixedcount_print(struct seq_file *m, |
| 287 | pg_data_t *pgdat, struct zone *zone) |
| 288 | { |
| 289 | struct page *page; |
| 290 | struct page_ext *page_ext; |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 291 | struct page_owner *page_owner; |
Joonsoo Kim | e2f612e | 2016-10-07 16:58:21 -0700 | [diff] [blame] | 292 | unsigned long pfn = zone->zone_start_pfn, block_end_pfn; |
| 293 | unsigned long end_pfn = pfn + zone->spanned_pages; |
| 294 | unsigned long count[MIGRATE_TYPES] = { 0, }; |
| 295 | int pageblock_mt, page_mt; |
| 296 | int i; |
| 297 | |
| 298 | /* Scan block by block. First and last block may be incomplete */ |
| 299 | pfn = zone->zone_start_pfn; |
| 300 | |
| 301 | /* |
| 302 | * Walk the zone in pageblock_nr_pages steps. If a page block spans |
| 303 | * a zone boundary, it will be double counted between zones. This does |
| 304 | * not matter as the mixed block count will still be correct |
| 305 | */ |
| 306 | for (; pfn < end_pfn; ) { |
Qian Cai | f712e30 | 2019-10-18 20:19:29 -0700 | [diff] [blame] | 307 | page = pfn_to_online_page(pfn); |
| 308 | if (!page) { |
Joonsoo Kim | e2f612e | 2016-10-07 16:58:21 -0700 | [diff] [blame] | 309 | pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); |
| 310 | continue; |
| 311 | } |
| 312 | |
| 313 | block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); |
| 314 | block_end_pfn = min(block_end_pfn, end_pfn); |
| 315 | |
Joonsoo Kim | e2f612e | 2016-10-07 16:58:21 -0700 | [diff] [blame] | 316 | pageblock_mt = get_pageblock_migratetype(page); |
| 317 | |
| 318 | for (; pfn < block_end_pfn; pfn++) { |
| 319 | if (!pfn_valid_within(pfn)) |
| 320 | continue; |
| 321 | |
Qian Cai | f712e30 | 2019-10-18 20:19:29 -0700 | [diff] [blame] | 322 | /* The pageblock is online, no need to recheck. */ |
Joonsoo Kim | e2f612e | 2016-10-07 16:58:21 -0700 | [diff] [blame] | 323 | page = pfn_to_page(pfn); |
| 324 | |
| 325 | if (page_zone(page) != zone) |
| 326 | continue; |
| 327 | |
| 328 | if (PageBuddy(page)) { |
Vinayak Menon | 727c080 | 2017-07-10 15:49:17 -0700 | [diff] [blame] | 329 | unsigned long freepage_order; |
| 330 | |
| 331 | freepage_order = page_order_unsafe(page); |
| 332 | if (freepage_order < MAX_ORDER) |
| 333 | pfn += (1UL << freepage_order) - 1; |
Joonsoo Kim | e2f612e | 2016-10-07 16:58:21 -0700 | [diff] [blame] | 334 | continue; |
| 335 | } |
| 336 | |
| 337 | if (PageReserved(page)) |
| 338 | continue; |
| 339 | |
| 340 | page_ext = lookup_page_ext(page); |
| 341 | if (unlikely(!page_ext)) |
| 342 | continue; |
| 343 | |
| 344 | if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) |
| 345 | continue; |
| 346 | |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 347 | page_owner = get_page_owner(page_ext); |
| 348 | page_mt = gfpflags_to_migratetype( |
| 349 | page_owner->gfp_mask); |
Joonsoo Kim | e2f612e | 2016-10-07 16:58:21 -0700 | [diff] [blame] | 350 | if (pageblock_mt != page_mt) { |
| 351 | if (is_migrate_cma(pageblock_mt)) |
| 352 | count[MIGRATE_MOVABLE]++; |
| 353 | else |
| 354 | count[pageblock_mt]++; |
| 355 | |
| 356 | pfn = block_end_pfn; |
| 357 | break; |
| 358 | } |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 359 | pfn += (1UL << page_owner->order) - 1; |
Joonsoo Kim | e2f612e | 2016-10-07 16:58:21 -0700 | [diff] [blame] | 360 | } |
| 361 | } |
| 362 | |
| 363 | /* Print counts */ |
| 364 | seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); |
| 365 | for (i = 0; i < MIGRATE_TYPES; i++) |
| 366 | seq_printf(m, "%12lu ", count[i]); |
| 367 | seq_putc(m, '\n'); |
| 368 | } |
| 369 | |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 370 | static ssize_t |
| 371 | print_page_owner(char __user *buf, size_t count, unsigned long pfn, |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 372 | struct page *page, struct page_owner *page_owner, |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 373 | depot_stack_handle_t handle) |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 374 | { |
| 375 | int ret; |
| 376 | int pageblock_mt, page_mt; |
| 377 | char *kbuf; |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 378 | unsigned long entries[PAGE_OWNER_STACK_DEPTH]; |
Sergei Rogachev | 94f759d6 | 2015-02-11 15:28:34 -0800 | [diff] [blame] | 379 | struct stack_trace trace = { |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 380 | .nr_entries = 0, |
| 381 | .entries = entries, |
| 382 | .max_entries = PAGE_OWNER_STACK_DEPTH, |
| 383 | .skip = 0 |
Sergei Rogachev | 94f759d6 | 2015-02-11 15:28:34 -0800 | [diff] [blame] | 384 | }; |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 385 | |
| 386 | kbuf = kmalloc(count, GFP_KERNEL); |
| 387 | if (!kbuf) |
| 388 | return -ENOMEM; |
| 389 | |
| 390 | ret = snprintf(kbuf, count, |
Liam Mark | ea0195f | 2017-12-21 14:27:20 -0800 | [diff] [blame] | 391 | "Page allocated via order %u, mask %#x(%pGg), pid %d, ts %llu ns\n", |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 392 | page_owner->order, page_owner->gfp_mask, |
Liam Mark | ea0195f | 2017-12-21 14:27:20 -0800 | [diff] [blame] | 393 | &page_owner->gfp_mask, page_owner->pid, |
| 394 | page_owner->ts_nsec); |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 395 | |
| 396 | if (ret >= count) |
| 397 | goto err; |
| 398 | |
| 399 | /* Print information relevant to grouping pages by mobility */ |
Mel Gorman | 0b423ca | 2016-05-19 17:14:27 -0700 | [diff] [blame] | 400 | pageblock_mt = get_pageblock_migratetype(page); |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 401 | page_mt = gfpflags_to_migratetype(page_owner->gfp_mask); |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 402 | ret += snprintf(kbuf + ret, count - ret, |
Vlastimil Babka | 60f3035 | 2016-03-15 14:56:08 -0700 | [diff] [blame] | 403 | "PFN %lu type %s Block %lu type %s Flags %#lx(%pGp)\n", |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 404 | pfn, |
Vlastimil Babka | 60f3035 | 2016-03-15 14:56:08 -0700 | [diff] [blame] | 405 | migratetype_names[page_mt], |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 406 | pfn >> pageblock_order, |
Vlastimil Babka | 60f3035 | 2016-03-15 14:56:08 -0700 | [diff] [blame] | 407 | migratetype_names[pageblock_mt], |
| 408 | page->flags, &page->flags); |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 409 | |
| 410 | if (ret >= count) |
| 411 | goto err; |
| 412 | |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 413 | depot_fetch_stack(handle, &trace); |
Sergei Rogachev | 94f759d6 | 2015-02-11 15:28:34 -0800 | [diff] [blame] | 414 | ret += snprint_stack_trace(kbuf + ret, count - ret, &trace, 0); |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 415 | if (ret >= count) |
| 416 | goto err; |
| 417 | |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 418 | if (page_owner->last_migrate_reason != -1) { |
Vlastimil Babka | 7cd12b4 | 2016-03-15 14:56:18 -0700 | [diff] [blame] | 419 | ret += snprintf(kbuf + ret, count - ret, |
| 420 | "Page has been migrated, last migrate reason: %s\n", |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 421 | migrate_reason_names[page_owner->last_migrate_reason]); |
Vlastimil Babka | 7cd12b4 | 2016-03-15 14:56:18 -0700 | [diff] [blame] | 422 | if (ret >= count) |
| 423 | goto err; |
| 424 | } |
| 425 | |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 426 | ret += snprintf(kbuf + ret, count - ret, "\n"); |
| 427 | if (ret >= count) |
| 428 | goto err; |
| 429 | |
| 430 | if (copy_to_user(buf, kbuf, ret)) |
| 431 | ret = -EFAULT; |
| 432 | |
| 433 | kfree(kbuf); |
| 434 | return ret; |
| 435 | |
| 436 | err: |
| 437 | kfree(kbuf); |
| 438 | return -ENOMEM; |
| 439 | } |
| 440 | |
Vlastimil Babka | 4e46211 | 2016-03-15 14:56:21 -0700 | [diff] [blame] | 441 | void __dump_page_owner(struct page *page) |
| 442 | { |
| 443 | struct page_ext *page_ext = lookup_page_ext(page); |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 444 | struct page_owner *page_owner; |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 445 | unsigned long entries[PAGE_OWNER_STACK_DEPTH]; |
Vlastimil Babka | 4e46211 | 2016-03-15 14:56:21 -0700 | [diff] [blame] | 446 | struct stack_trace trace = { |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 447 | .nr_entries = 0, |
| 448 | .entries = entries, |
| 449 | .max_entries = PAGE_OWNER_STACK_DEPTH, |
| 450 | .skip = 0 |
Vlastimil Babka | 4e46211 | 2016-03-15 14:56:21 -0700 | [diff] [blame] | 451 | }; |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 452 | depot_stack_handle_t handle; |
Sudip Mukherjee | 8285027 | 2016-06-24 14:50:24 -0700 | [diff] [blame] | 453 | gfp_t gfp_mask; |
| 454 | int mt; |
Vlastimil Babka | 4e46211 | 2016-03-15 14:56:21 -0700 | [diff] [blame] | 455 | |
Yang Shi | f86e427 | 2016-06-03 14:55:38 -0700 | [diff] [blame] | 456 | if (unlikely(!page_ext)) { |
| 457 | pr_alert("There is not page extension available.\n"); |
| 458 | return; |
| 459 | } |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 460 | |
| 461 | page_owner = get_page_owner(page_ext); |
| 462 | gfp_mask = page_owner->gfp_mask; |
Sudip Mukherjee | 8285027 | 2016-06-24 14:50:24 -0700 | [diff] [blame] | 463 | mt = gfpflags_to_migratetype(gfp_mask); |
Yang Shi | f86e427 | 2016-06-03 14:55:38 -0700 | [diff] [blame] | 464 | |
Vlastimil Babka | 4e46211 | 2016-03-15 14:56:21 -0700 | [diff] [blame] | 465 | if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) { |
| 466 | pr_alert("page_owner info is not active (free page?)\n"); |
| 467 | return; |
| 468 | } |
| 469 | |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 470 | handle = READ_ONCE(page_owner->handle); |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 471 | if (!handle) { |
| 472 | pr_alert("page_owner info is not active (free page?)\n"); |
| 473 | return; |
| 474 | } |
| 475 | |
| 476 | depot_fetch_stack(handle, &trace); |
Liam Mark | ea0195f | 2017-12-21 14:27:20 -0800 | [diff] [blame] | 477 | pr_alert("page allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, ts %llu ns\n", |
| 478 | page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask, |
| 479 | page_owner->pid, page_owner->ts_nsec); |
Vlastimil Babka | 4e46211 | 2016-03-15 14:56:21 -0700 | [diff] [blame] | 480 | print_stack_trace(&trace, 0); |
| 481 | |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 482 | if (page_owner->last_migrate_reason != -1) |
Vlastimil Babka | 4e46211 | 2016-03-15 14:56:21 -0700 | [diff] [blame] | 483 | pr_alert("page has been migrated, last migrate reason: %s\n", |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 484 | migrate_reason_names[page_owner->last_migrate_reason]); |
Vlastimil Babka | 4e46211 | 2016-03-15 14:56:21 -0700 | [diff] [blame] | 485 | } |
| 486 | |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 487 | static ssize_t |
| 488 | read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos) |
| 489 | { |
| 490 | unsigned long pfn; |
| 491 | struct page *page; |
| 492 | struct page_ext *page_ext; |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 493 | struct page_owner *page_owner; |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 494 | depot_stack_handle_t handle; |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 495 | |
Vlastimil Babka | 7dd80b8 | 2016-03-15 14:56:12 -0700 | [diff] [blame] | 496 | if (!static_branch_unlikely(&page_owner_inited)) |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 497 | return -EINVAL; |
| 498 | |
| 499 | page = NULL; |
| 500 | pfn = min_low_pfn + *ppos; |
| 501 | |
| 502 | /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */ |
| 503 | while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0) |
| 504 | pfn++; |
| 505 | |
| 506 | drain_all_pages(NULL); |
| 507 | |
| 508 | /* Find an allocated page */ |
| 509 | for (; pfn < max_pfn; pfn++) { |
| 510 | /* |
| 511 | * If the new page is in a new MAX_ORDER_NR_PAGES area, |
| 512 | * validate the area as existing, skip it if not |
| 513 | */ |
| 514 | if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) { |
| 515 | pfn += MAX_ORDER_NR_PAGES - 1; |
| 516 | continue; |
| 517 | } |
| 518 | |
| 519 | /* Check for holes within a MAX_ORDER area */ |
| 520 | if (!pfn_valid_within(pfn)) |
| 521 | continue; |
| 522 | |
| 523 | page = pfn_to_page(pfn); |
| 524 | if (PageBuddy(page)) { |
| 525 | unsigned long freepage_order = page_order_unsafe(page); |
| 526 | |
| 527 | if (freepage_order < MAX_ORDER) |
| 528 | pfn += (1UL << freepage_order) - 1; |
| 529 | continue; |
| 530 | } |
| 531 | |
| 532 | page_ext = lookup_page_ext(page); |
Yang Shi | f86e427 | 2016-06-03 14:55:38 -0700 | [diff] [blame] | 533 | if (unlikely(!page_ext)) |
| 534 | continue; |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 535 | |
| 536 | /* |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 537 | * Some pages could be missed by concurrent allocation or free, |
| 538 | * because we don't hold the zone lock. |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 539 | */ |
| 540 | if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) |
| 541 | continue; |
| 542 | |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 543 | page_owner = get_page_owner(page_ext); |
| 544 | |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 545 | /* |
| 546 | * Access to page_ext->handle isn't synchronous so we should |
| 547 | * be careful to access it. |
| 548 | */ |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 549 | handle = READ_ONCE(page_owner->handle); |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 550 | if (!handle) |
| 551 | continue; |
| 552 | |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 553 | /* Record the next PFN to read in the file offset */ |
| 554 | *ppos = (pfn - min_low_pfn) + 1; |
| 555 | |
Joonsoo Kim | f2ca0b5 | 2016-07-26 15:23:55 -0700 | [diff] [blame] | 556 | return print_page_owner(buf, count, pfn, page, |
Joonsoo Kim | 9300d8d | 2016-10-07 16:58:30 -0700 | [diff] [blame] | 557 | page_owner, handle); |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 558 | } |
| 559 | |
| 560 | return 0; |
| 561 | } |
| 562 | |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 563 | static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone) |
| 564 | { |
Oscar Salvador | 6787c1d | 2018-01-31 16:20:11 -0800 | [diff] [blame] | 565 | unsigned long pfn = zone->zone_start_pfn; |
| 566 | unsigned long end_pfn = zone_end_pfn(zone); |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 567 | unsigned long count = 0; |
| 568 | |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 569 | /* |
| 570 | * Walk the zone in pageblock_nr_pages steps. If a page block spans |
| 571 | * a zone boundary, it will be double counted between zones. This does |
| 572 | * not matter as the mixed block count will still be correct |
| 573 | */ |
| 574 | for (; pfn < end_pfn; ) { |
Oscar Salvador | 6787c1d | 2018-01-31 16:20:11 -0800 | [diff] [blame] | 575 | unsigned long block_end_pfn; |
| 576 | |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 577 | if (!pfn_valid(pfn)) { |
| 578 | pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES); |
| 579 | continue; |
| 580 | } |
| 581 | |
| 582 | block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages); |
| 583 | block_end_pfn = min(block_end_pfn, end_pfn); |
| 584 | |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 585 | for (; pfn < block_end_pfn; pfn++) { |
Oscar Salvador | 6787c1d | 2018-01-31 16:20:11 -0800 | [diff] [blame] | 586 | struct page *page; |
| 587 | struct page_ext *page_ext; |
| 588 | |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 589 | if (!pfn_valid_within(pfn)) |
| 590 | continue; |
| 591 | |
| 592 | page = pfn_to_page(pfn); |
| 593 | |
Joonsoo Kim | 9d43f5a | 2016-05-19 17:12:13 -0700 | [diff] [blame] | 594 | if (page_zone(page) != zone) |
| 595 | continue; |
| 596 | |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 597 | /* |
Vlastimil Babka | 1090302 | 2017-09-06 16:20:51 -0700 | [diff] [blame] | 598 | * To avoid having to grab zone->lock, be a little |
| 599 | * careful when reading buddy page order. The only |
| 600 | * danger is that we skip too much and potentially miss |
| 601 | * some early allocated pages, which is better than |
| 602 | * heavy lock contention. |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 603 | */ |
| 604 | if (PageBuddy(page)) { |
Vlastimil Babka | 1090302 | 2017-09-06 16:20:51 -0700 | [diff] [blame] | 605 | unsigned long order = page_order_unsafe(page); |
| 606 | |
| 607 | if (order > 0 && order < MAX_ORDER) |
| 608 | pfn += (1UL << order) - 1; |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 609 | continue; |
| 610 | } |
| 611 | |
| 612 | if (PageReserved(page)) |
| 613 | continue; |
| 614 | |
| 615 | page_ext = lookup_page_ext(page); |
Yang Shi | f86e427 | 2016-06-03 14:55:38 -0700 | [diff] [blame] | 616 | if (unlikely(!page_ext)) |
| 617 | continue; |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 618 | |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 619 | /* Maybe overlapping zone */ |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 620 | if (test_bit(PAGE_EXT_OWNER, &page_ext->flags)) |
| 621 | continue; |
| 622 | |
| 623 | /* Found early allocated page */ |
Vlastimil Babka | dab4ead | 2017-09-06 16:20:44 -0700 | [diff] [blame] | 624 | __set_page_owner_handle(page_ext, early_handle, 0, 0); |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 625 | count++; |
| 626 | } |
Vlastimil Babka | 1090302 | 2017-09-06 16:20:51 -0700 | [diff] [blame] | 627 | cond_resched(); |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 628 | } |
| 629 | |
| 630 | pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n", |
| 631 | pgdat->node_id, zone->name, count); |
| 632 | } |
| 633 | |
| 634 | static void init_zones_in_node(pg_data_t *pgdat) |
| 635 | { |
| 636 | struct zone *zone; |
| 637 | struct zone *node_zones = pgdat->node_zones; |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 638 | |
| 639 | for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { |
| 640 | if (!populated_zone(zone)) |
| 641 | continue; |
| 642 | |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 643 | init_pages_in_zone(pgdat, zone); |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 644 | } |
| 645 | } |
| 646 | |
| 647 | static void init_early_allocated_pages(void) |
| 648 | { |
| 649 | pg_data_t *pgdat; |
| 650 | |
Joonsoo Kim | 61cf5fe | 2014-12-12 16:56:04 -0800 | [diff] [blame] | 651 | for_each_online_pgdat(pgdat) |
| 652 | init_zones_in_node(pgdat); |
| 653 | } |
| 654 | |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 655 | static const struct file_operations proc_page_owner_operations = { |
| 656 | .read = read_page_owner, |
| 657 | }; |
| 658 | |
| 659 | static int __init pageowner_init(void) |
| 660 | { |
| 661 | struct dentry *dentry; |
| 662 | |
Vlastimil Babka | 7dd80b8 | 2016-03-15 14:56:12 -0700 | [diff] [blame] | 663 | if (!static_branch_unlikely(&page_owner_inited)) { |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 664 | pr_info("page_owner is disabled\n"); |
| 665 | return 0; |
| 666 | } |
| 667 | |
Joe Perches | 0825a6f | 2018-06-14 15:27:58 -0700 | [diff] [blame] | 668 | dentry = debugfs_create_file("page_owner", 0400, NULL, |
| 669 | NULL, &proc_page_owner_operations); |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 670 | |
Vasyl Gomonovych | 8e33771 | 2018-01-31 16:16:48 -0800 | [diff] [blame] | 671 | return PTR_ERR_OR_ZERO(dentry); |
Joonsoo Kim | 48c96a3 | 2014-12-12 16:56:01 -0800 | [diff] [blame] | 672 | } |
Paul Gortmaker | 44c5af9 | 2015-05-01 21:57:34 -0400 | [diff] [blame] | 673 | late_initcall(pageowner_init) |