Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1 | /* memcontrol.h - Memory Controller |
| 2 | * |
| 3 | * Copyright IBM Corporation, 2007 |
| 4 | * Author Balbir Singh <balbir@linux.vnet.ibm.com> |
| 5 | * |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 6 | * Copyright 2007 OpenVZ SWsoft Inc |
| 7 | * Author: Pavel Emelianov <xemul@openvz.org> |
| 8 | * |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 9 | * This program is free software; you can redistribute it and/or modify |
| 10 | * it under the terms of the GNU General Public License as published by |
| 11 | * the Free Software Foundation; either version 2 of the License, or |
| 12 | * (at your option) any later version. |
| 13 | * |
| 14 | * This program is distributed in the hope that it will be useful, |
| 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 17 | * GNU General Public License for more details. |
| 18 | */ |
| 19 | |
| 20 | #ifndef _LINUX_MEMCONTROL_H |
| 21 | #define _LINUX_MEMCONTROL_H |
Hirokazu Takahashi | f8d66542 | 2009-01-07 18:08:02 -0800 | [diff] [blame] | 22 | #include <linux/cgroup.h> |
Ying Han | 456f998 | 2011-05-26 16:25:38 -0700 | [diff] [blame] | 23 | #include <linux/vm_event_item.h> |
Glauber Costa | 7ae1e1d | 2012-12-18 14:21:56 -0800 | [diff] [blame] | 24 | #include <linux/hardirq.h> |
Glauber Costa | a8964b9 | 2012-12-18 14:22:09 -0800 | [diff] [blame] | 25 | #include <linux/jump_label.h> |
Ying Han | 456f998 | 2011-05-26 16:25:38 -0700 | [diff] [blame] | 26 | |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 27 | struct mem_cgroup; |
| 28 | struct page_cgroup; |
Balbir Singh | 8697d33 | 2008-02-07 00:13:59 -0800 | [diff] [blame] | 29 | struct page; |
| 30 | struct mm_struct; |
Glauber Costa | 2633d7a | 2012-12-18 14:22:34 -0800 | [diff] [blame] | 31 | struct kmem_cache; |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 32 | |
Greg Thelen | 2a7106f | 2011-01-13 15:47:37 -0800 | [diff] [blame] | 33 | /* Stats that can be updated by kernel. */ |
| 34 | enum mem_cgroup_page_stat_item { |
| 35 | MEMCG_NR_FILE_MAPPED, /* # of pages charged as file rss */ |
| 36 | }; |
| 37 | |
Johannes Weiner | 5660048 | 2012-01-12 17:17:59 -0800 | [diff] [blame] | 38 | struct mem_cgroup_reclaim_cookie { |
| 39 | struct zone *zone; |
| 40 | int priority; |
| 41 | unsigned int generation; |
| 42 | }; |
| 43 | |
Andrew Morton | c255a45 | 2012-07-31 16:43:02 -0700 | [diff] [blame] | 44 | #ifdef CONFIG_MEMCG |
KAMEZAWA Hiroyuki | 2c26fdd | 2009-01-07 18:08:10 -0800 | [diff] [blame] | 45 | /* |
| 46 | * All "charge" functions with gfp_mask should use GFP_KERNEL or |
| 47 | * (gfp_mask & GFP_RECLAIM_MASK). In current implementatin, memcg doesn't |
| 48 | * alloc memory but reclaims memory from all available zones. So, "where I want |
| 49 | * memory from" bits of gfp_mask has no meaning. So any bits of that field is |
| 50 | * available but adding a rule is better. charge functions' gfp_mask should |
| 51 | * be set to GFP_KERNEL or gfp_mask & GFP_RECLAIM_MASK for avoiding ambiguous |
| 52 | * codes. |
| 53 | * (Of course, if memcg does memory allocation in future, GFP_KERNEL is sane.) |
| 54 | */ |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 55 | |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 56 | extern int mem_cgroup_newpage_charge(struct page *page, struct mm_struct *mm, |
Balbir Singh | e1a1cd5 | 2008-02-07 00:14:02 -0800 | [diff] [blame] | 57 | gfp_t gfp_mask); |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 58 | /* for swap handling */ |
KAMEZAWA Hiroyuki | 8c7c6e34 | 2009-01-07 18:08:00 -0800 | [diff] [blame] | 59 | extern int mem_cgroup_try_charge_swapin(struct mm_struct *mm, |
Johannes Weiner | 72835c8 | 2012-01-12 17:18:32 -0800 | [diff] [blame] | 60 | struct page *page, gfp_t mask, struct mem_cgroup **memcgp); |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 61 | extern void mem_cgroup_commit_charge_swapin(struct page *page, |
Johannes Weiner | 72835c8 | 2012-01-12 17:18:32 -0800 | [diff] [blame] | 62 | struct mem_cgroup *memcg); |
| 63 | extern void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg); |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 64 | |
Hugh Dickins | 8289546 | 2008-03-04 14:29:08 -0800 | [diff] [blame] | 65 | extern int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, |
| 66 | gfp_t gfp_mask); |
Johannes Weiner | 925b767 | 2012-01-12 17:18:15 -0800 | [diff] [blame] | 67 | |
| 68 | struct lruvec *mem_cgroup_zone_lruvec(struct zone *, struct mem_cgroup *); |
Hugh Dickins | fa9add6 | 2012-05-29 15:07:09 -0700 | [diff] [blame] | 69 | struct lruvec *mem_cgroup_page_lruvec(struct page *, struct zone *); |
KAMEZAWA Hiroyuki | 569b846 | 2009-12-15 16:47:03 -0800 | [diff] [blame] | 70 | |
| 71 | /* For coalescing uncharge for reducing memcg' overhead*/ |
| 72 | extern void mem_cgroup_uncharge_start(void); |
| 73 | extern void mem_cgroup_uncharge_end(void); |
| 74 | |
Balbir Singh | 3c541e1 | 2008-02-07 00:14:41 -0800 | [diff] [blame] | 75 | extern void mem_cgroup_uncharge_page(struct page *page); |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 76 | extern void mem_cgroup_uncharge_cache_page(struct page *page); |
KAMEZAWA Hiroyuki | c9b0ed5 | 2008-07-25 01:47:15 -0700 | [diff] [blame] | 77 | |
Johannes Weiner | c3ac9a8 | 2012-05-29 15:06:25 -0700 | [diff] [blame] | 78 | bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg, |
| 79 | struct mem_cgroup *memcg); |
Raghavendra K T | c0ff4b8 | 2011-11-02 13:38:15 -0700 | [diff] [blame] | 80 | int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg); |
David Rientjes | 3062fc6 | 2008-02-07 00:14:03 -0800 | [diff] [blame] | 81 | |
Wu Fengguang | e42d9d5 | 2009-12-16 12:19:59 +0100 | [diff] [blame] | 82 | extern struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page); |
Balbir Singh | cf475ad | 2008-04-29 01:00:16 -0700 | [diff] [blame] | 83 | extern struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p); |
KOSAKI Motohiro | a433658 | 2011-06-15 15:08:13 -0700 | [diff] [blame] | 84 | extern struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm); |
Balbir Singh | cf475ad | 2008-04-29 01:00:16 -0700 | [diff] [blame] | 85 | |
Glauber Costa | e1aab16 | 2011-12-11 21:47:03 +0000 | [diff] [blame] | 86 | extern struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg); |
Glauber Costa | d1a4c0b | 2011-12-11 21:47:04 +0000 | [diff] [blame] | 87 | extern struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont); |
Glauber Costa | e1aab16 | 2011-12-11 21:47:03 +0000 | [diff] [blame] | 88 | |
Lai Jiangshan | 2e4d409 | 2009-01-07 18:08:07 -0800 | [diff] [blame] | 89 | static inline |
Johannes Weiner | 587af30 | 2012-10-08 16:34:12 -0700 | [diff] [blame] | 90 | bool mm_match_cgroup(const struct mm_struct *mm, const struct mem_cgroup *memcg) |
Lai Jiangshan | 2e4d409 | 2009-01-07 18:08:07 -0800 | [diff] [blame] | 91 | { |
Johannes Weiner | 587af30 | 2012-10-08 16:34:12 -0700 | [diff] [blame] | 92 | struct mem_cgroup *task_memcg; |
| 93 | bool match; |
Johannes Weiner | c3ac9a8 | 2012-05-29 15:06:25 -0700 | [diff] [blame] | 94 | |
Lai Jiangshan | 2e4d409 | 2009-01-07 18:08:07 -0800 | [diff] [blame] | 95 | rcu_read_lock(); |
Johannes Weiner | 587af30 | 2012-10-08 16:34:12 -0700 | [diff] [blame] | 96 | task_memcg = mem_cgroup_from_task(rcu_dereference(mm->owner)); |
| 97 | match = __mem_cgroup_same_or_subtree(memcg, task_memcg); |
Lai Jiangshan | 2e4d409 | 2009-01-07 18:08:07 -0800 | [diff] [blame] | 98 | rcu_read_unlock(); |
Johannes Weiner | c3ac9a8 | 2012-05-29 15:06:25 -0700 | [diff] [blame] | 99 | return match; |
Lai Jiangshan | 2e4d409 | 2009-01-07 18:08:07 -0800 | [diff] [blame] | 100 | } |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 101 | |
Raghavendra K T | c0ff4b8 | 2011-11-02 13:38:15 -0700 | [diff] [blame] | 102 | extern struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg); |
Wu Fengguang | d324236 | 2009-12-16 12:19:59 +0100 | [diff] [blame] | 103 | |
Johannes Weiner | 0030f53 | 2012-07-31 16:45:25 -0700 | [diff] [blame] | 104 | extern void |
| 105 | mem_cgroup_prepare_migration(struct page *page, struct page *newpage, |
| 106 | struct mem_cgroup **memcgp); |
Raghavendra K T | c0ff4b8 | 2011-11-02 13:38:15 -0700 | [diff] [blame] | 107 | extern void mem_cgroup_end_migration(struct mem_cgroup *memcg, |
Daisuke Nishimura | 50de1dd | 2011-01-13 15:47:43 -0800 | [diff] [blame] | 108 | struct page *oldpage, struct page *newpage, bool migration_ok); |
KAMEZAWA Hiroyuki | ae41be3 | 2008-02-07 00:14:10 -0800 | [diff] [blame] | 109 | |
Johannes Weiner | 5660048 | 2012-01-12 17:17:59 -0800 | [diff] [blame] | 110 | struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *, |
| 111 | struct mem_cgroup *, |
| 112 | struct mem_cgroup_reclaim_cookie *); |
| 113 | void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *); |
| 114 | |
KAMEZAWA Hiroyuki | 58ae83d | 2008-02-07 00:14:32 -0800 | [diff] [blame] | 115 | /* |
| 116 | * For memory reclaim. |
| 117 | */ |
Konstantin Khlebnikov | c56d5c7 | 2012-05-29 15:07:00 -0700 | [diff] [blame] | 118 | int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec); |
| 119 | int mem_cgroup_inactive_file_is_low(struct lruvec *lruvec); |
Ying Han | 889976d | 2011-05-26 16:25:33 -0700 | [diff] [blame] | 120 | int mem_cgroup_select_victim_node(struct mem_cgroup *memcg); |
Hugh Dickins | 4d7dcca | 2012-05-29 15:07:08 -0700 | [diff] [blame] | 121 | unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list); |
Hugh Dickins | fa9add6 | 2012-05-29 15:07:09 -0700 | [diff] [blame] | 122 | void mem_cgroup_update_lru_size(struct lruvec *, enum lru_list, int); |
Balbir Singh | e222432 | 2009-04-02 16:57:39 -0700 | [diff] [blame] | 123 | extern void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, |
| 124 | struct task_struct *p); |
KAMEZAWA Hiroyuki | ab936cb | 2012-01-12 17:17:44 -0800 | [diff] [blame] | 125 | extern void mem_cgroup_replace_page_cache(struct page *oldpage, |
| 126 | struct page *newpage); |
KAMEZAWA Hiroyuki | 58ae83d | 2008-02-07 00:14:32 -0800 | [diff] [blame] | 127 | |
Andrew Morton | c255a45 | 2012-07-31 16:43:02 -0700 | [diff] [blame] | 128 | #ifdef CONFIG_MEMCG_SWAP |
KAMEZAWA Hiroyuki | c077719 | 2009-01-07 18:07:57 -0800 | [diff] [blame] | 129 | extern int do_swap_account; |
| 130 | #endif |
Hirokazu Takahashi | f8d66542 | 2009-01-07 18:08:02 -0800 | [diff] [blame] | 131 | |
| 132 | static inline bool mem_cgroup_disabled(void) |
| 133 | { |
| 134 | if (mem_cgroup_subsys.disabled) |
| 135 | return true; |
| 136 | return false; |
| 137 | } |
| 138 | |
KAMEZAWA Hiroyuki | 89c06bd | 2012-03-21 16:34:25 -0700 | [diff] [blame] | 139 | void __mem_cgroup_begin_update_page_stat(struct page *page, bool *locked, |
| 140 | unsigned long *flags); |
| 141 | |
KAMEZAWA Hiroyuki | 4331f7d | 2012-03-21 16:34:26 -0700 | [diff] [blame] | 142 | extern atomic_t memcg_moving; |
| 143 | |
KAMEZAWA Hiroyuki | 89c06bd | 2012-03-21 16:34:25 -0700 | [diff] [blame] | 144 | static inline void mem_cgroup_begin_update_page_stat(struct page *page, |
| 145 | bool *locked, unsigned long *flags) |
| 146 | { |
| 147 | if (mem_cgroup_disabled()) |
| 148 | return; |
| 149 | rcu_read_lock(); |
| 150 | *locked = false; |
KAMEZAWA Hiroyuki | 4331f7d | 2012-03-21 16:34:26 -0700 | [diff] [blame] | 151 | if (atomic_read(&memcg_moving)) |
| 152 | __mem_cgroup_begin_update_page_stat(page, locked, flags); |
KAMEZAWA Hiroyuki | 89c06bd | 2012-03-21 16:34:25 -0700 | [diff] [blame] | 153 | } |
| 154 | |
| 155 | void __mem_cgroup_end_update_page_stat(struct page *page, |
| 156 | unsigned long *flags); |
| 157 | static inline void mem_cgroup_end_update_page_stat(struct page *page, |
| 158 | bool *locked, unsigned long *flags) |
| 159 | { |
| 160 | if (mem_cgroup_disabled()) |
| 161 | return; |
| 162 | if (*locked) |
| 163 | __mem_cgroup_end_update_page_stat(page, flags); |
| 164 | rcu_read_unlock(); |
| 165 | } |
| 166 | |
Greg Thelen | 2a7106f | 2011-01-13 15:47:37 -0800 | [diff] [blame] | 167 | void mem_cgroup_update_page_stat(struct page *page, |
| 168 | enum mem_cgroup_page_stat_item idx, |
| 169 | int val); |
| 170 | |
| 171 | static inline void mem_cgroup_inc_page_stat(struct page *page, |
| 172 | enum mem_cgroup_page_stat_item idx) |
| 173 | { |
| 174 | mem_cgroup_update_page_stat(page, idx, 1); |
| 175 | } |
| 176 | |
| 177 | static inline void mem_cgroup_dec_page_stat(struct page *page, |
| 178 | enum mem_cgroup_page_stat_item idx) |
| 179 | { |
| 180 | mem_cgroup_update_page_stat(page, idx, -1); |
| 181 | } |
| 182 | |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 183 | unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, |
Ying Han | 0ae5e89 | 2011-05-26 16:25:25 -0700 | [diff] [blame] | 184 | gfp_t gfp_mask, |
| 185 | unsigned long *total_scanned); |
David Rientjes | a63d83f | 2010-08-09 17:19:46 -0700 | [diff] [blame] | 186 | |
David Rientjes | 68ae564 | 2012-12-12 13:51:57 -0800 | [diff] [blame] | 187 | void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx); |
| 188 | static inline void mem_cgroup_count_vm_event(struct mm_struct *mm, |
| 189 | enum vm_event_item idx) |
| 190 | { |
| 191 | if (mem_cgroup_disabled()) |
| 192 | return; |
| 193 | __mem_cgroup_count_vm_event(mm, idx); |
| 194 | } |
KAMEZAWA Hiroyuki | ca3e021 | 2011-01-20 14:44:24 -0800 | [diff] [blame] | 195 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
KAMEZAWA Hiroyuki | e94c8a9 | 2012-01-12 17:18:20 -0800 | [diff] [blame] | 196 | void mem_cgroup_split_huge_fixup(struct page *head); |
KAMEZAWA Hiroyuki | ca3e021 | 2011-01-20 14:44:24 -0800 | [diff] [blame] | 197 | #endif |
| 198 | |
Daisuke Nishimura | f212ad7 | 2011-03-23 16:42:25 -0700 | [diff] [blame] | 199 | #ifdef CONFIG_DEBUG_VM |
| 200 | bool mem_cgroup_bad_page_check(struct page *page); |
| 201 | void mem_cgroup_print_bad_page(struct page *page); |
| 202 | #endif |
Andrew Morton | c255a45 | 2012-07-31 16:43:02 -0700 | [diff] [blame] | 203 | #else /* CONFIG_MEMCG */ |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 204 | struct mem_cgroup; |
| 205 | |
| 206 | static inline int mem_cgroup_newpage_charge(struct page *page, |
Hugh Dickins | 8289546 | 2008-03-04 14:29:08 -0800 | [diff] [blame] | 207 | struct mm_struct *mm, gfp_t gfp_mask) |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 208 | { |
| 209 | return 0; |
| 210 | } |
| 211 | |
Hugh Dickins | 8289546 | 2008-03-04 14:29:08 -0800 | [diff] [blame] | 212 | static inline int mem_cgroup_cache_charge(struct page *page, |
| 213 | struct mm_struct *mm, gfp_t gfp_mask) |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 214 | { |
Hugh Dickins | 8289546 | 2008-03-04 14:29:08 -0800 | [diff] [blame] | 215 | return 0; |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 216 | } |
| 217 | |
KAMEZAWA Hiroyuki | 8c7c6e34 | 2009-01-07 18:08:00 -0800 | [diff] [blame] | 218 | static inline int mem_cgroup_try_charge_swapin(struct mm_struct *mm, |
Johannes Weiner | 72835c8 | 2012-01-12 17:18:32 -0800 | [diff] [blame] | 219 | struct page *page, gfp_t gfp_mask, struct mem_cgroup **memcgp) |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 220 | { |
| 221 | return 0; |
| 222 | } |
| 223 | |
| 224 | static inline void mem_cgroup_commit_charge_swapin(struct page *page, |
Johannes Weiner | 72835c8 | 2012-01-12 17:18:32 -0800 | [diff] [blame] | 225 | struct mem_cgroup *memcg) |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 226 | { |
| 227 | } |
| 228 | |
Johannes Weiner | 72835c8 | 2012-01-12 17:18:32 -0800 | [diff] [blame] | 229 | static inline void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg) |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 230 | { |
| 231 | } |
| 232 | |
KAMEZAWA Hiroyuki | 569b846 | 2009-12-15 16:47:03 -0800 | [diff] [blame] | 233 | static inline void mem_cgroup_uncharge_start(void) |
| 234 | { |
| 235 | } |
| 236 | |
| 237 | static inline void mem_cgroup_uncharge_end(void) |
| 238 | { |
| 239 | } |
| 240 | |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 241 | static inline void mem_cgroup_uncharge_page(struct page *page) |
| 242 | { |
| 243 | } |
| 244 | |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 245 | static inline void mem_cgroup_uncharge_cache_page(struct page *page) |
| 246 | { |
| 247 | } |
| 248 | |
Johannes Weiner | 925b767 | 2012-01-12 17:18:15 -0800 | [diff] [blame] | 249 | static inline struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone, |
| 250 | struct mem_cgroup *memcg) |
| 251 | { |
| 252 | return &zone->lruvec; |
| 253 | } |
| 254 | |
Hugh Dickins | fa9add6 | 2012-05-29 15:07:09 -0700 | [diff] [blame] | 255 | static inline struct lruvec *mem_cgroup_page_lruvec(struct page *page, |
| 256 | struct zone *zone) |
Minchan Kim | 3f58a82 | 2011-03-22 16:32:53 -0700 | [diff] [blame] | 257 | { |
Johannes Weiner | 925b767 | 2012-01-12 17:18:15 -0800 | [diff] [blame] | 258 | return &zone->lruvec; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 259 | } |
| 260 | |
Wu Fengguang | e42d9d5 | 2009-12-16 12:19:59 +0100 | [diff] [blame] | 261 | static inline struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page) |
| 262 | { |
| 263 | return NULL; |
| 264 | } |
| 265 | |
KOSAKI Motohiro | a433658 | 2011-06-15 15:08:13 -0700 | [diff] [blame] | 266 | static inline struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm) |
| 267 | { |
| 268 | return NULL; |
| 269 | } |
| 270 | |
Johannes Weiner | 587af30 | 2012-10-08 16:34:12 -0700 | [diff] [blame] | 271 | static inline bool mm_match_cgroup(struct mm_struct *mm, |
Raghavendra K T | c0ff4b8 | 2011-11-02 13:38:15 -0700 | [diff] [blame] | 272 | struct mem_cgroup *memcg) |
Balbir Singh | bed7161 | 2008-02-07 00:14:01 -0800 | [diff] [blame] | 273 | { |
Johannes Weiner | 587af30 | 2012-10-08 16:34:12 -0700 | [diff] [blame] | 274 | return true; |
Balbir Singh | bed7161 | 2008-02-07 00:14:01 -0800 | [diff] [blame] | 275 | } |
| 276 | |
David Rientjes | 4c4a221 | 2008-02-07 00:14:06 -0800 | [diff] [blame] | 277 | static inline int task_in_mem_cgroup(struct task_struct *task, |
Raghavendra K T | c0ff4b8 | 2011-11-02 13:38:15 -0700 | [diff] [blame] | 278 | const struct mem_cgroup *memcg) |
David Rientjes | 4c4a221 | 2008-02-07 00:14:06 -0800 | [diff] [blame] | 279 | { |
| 280 | return 1; |
| 281 | } |
| 282 | |
Raghavendra K T | c0ff4b8 | 2011-11-02 13:38:15 -0700 | [diff] [blame] | 283 | static inline struct cgroup_subsys_state |
| 284 | *mem_cgroup_css(struct mem_cgroup *memcg) |
Wu Fengguang | d324236 | 2009-12-16 12:19:59 +0100 | [diff] [blame] | 285 | { |
| 286 | return NULL; |
| 287 | } |
| 288 | |
Johannes Weiner | 0030f53 | 2012-07-31 16:45:25 -0700 | [diff] [blame] | 289 | static inline void |
akpm@linux-foundation.org | ac39cf8 | 2010-05-26 14:42:46 -0700 | [diff] [blame] | 290 | mem_cgroup_prepare_migration(struct page *page, struct page *newpage, |
Johannes Weiner | 0030f53 | 2012-07-31 16:45:25 -0700 | [diff] [blame] | 291 | struct mem_cgroup **memcgp) |
KAMEZAWA Hiroyuki | ae41be3 | 2008-02-07 00:14:10 -0800 | [diff] [blame] | 292 | { |
KAMEZAWA Hiroyuki | ae41be3 | 2008-02-07 00:14:10 -0800 | [diff] [blame] | 293 | } |
| 294 | |
Raghavendra K T | c0ff4b8 | 2011-11-02 13:38:15 -0700 | [diff] [blame] | 295 | static inline void mem_cgroup_end_migration(struct mem_cgroup *memcg, |
Daisuke Nishimura | 50de1dd | 2011-01-13 15:47:43 -0800 | [diff] [blame] | 296 | struct page *oldpage, struct page *newpage, bool migration_ok) |
KAMEZAWA Hiroyuki | ae41be3 | 2008-02-07 00:14:10 -0800 | [diff] [blame] | 297 | { |
| 298 | } |
| 299 | |
Johannes Weiner | 5660048 | 2012-01-12 17:17:59 -0800 | [diff] [blame] | 300 | static inline struct mem_cgroup * |
| 301 | mem_cgroup_iter(struct mem_cgroup *root, |
| 302 | struct mem_cgroup *prev, |
| 303 | struct mem_cgroup_reclaim_cookie *reclaim) |
| 304 | { |
| 305 | return NULL; |
| 306 | } |
| 307 | |
| 308 | static inline void mem_cgroup_iter_break(struct mem_cgroup *root, |
| 309 | struct mem_cgroup *prev) |
| 310 | { |
| 311 | } |
| 312 | |
Hirokazu Takahashi | f8d66542 | 2009-01-07 18:08:02 -0800 | [diff] [blame] | 313 | static inline bool mem_cgroup_disabled(void) |
| 314 | { |
| 315 | return true; |
| 316 | } |
KAMEZAWA Hiroyuki | a636b32 | 2009-01-07 18:08:08 -0800 | [diff] [blame] | 317 | |
KOSAKI Motohiro | 14797e2 | 2009-01-07 18:08:18 -0800 | [diff] [blame] | 318 | static inline int |
Konstantin Khlebnikov | c56d5c7 | 2012-05-29 15:07:00 -0700 | [diff] [blame] | 319 | mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec) |
KOSAKI Motohiro | 14797e2 | 2009-01-07 18:08:18 -0800 | [diff] [blame] | 320 | { |
| 321 | return 1; |
| 322 | } |
| 323 | |
Rik van Riel | 56e49d2 | 2009-06-16 15:32:28 -0700 | [diff] [blame] | 324 | static inline int |
Konstantin Khlebnikov | c56d5c7 | 2012-05-29 15:07:00 -0700 | [diff] [blame] | 325 | mem_cgroup_inactive_file_is_low(struct lruvec *lruvec) |
Rik van Riel | 56e49d2 | 2009-06-16 15:32:28 -0700 | [diff] [blame] | 326 | { |
| 327 | return 1; |
| 328 | } |
| 329 | |
KOSAKI Motohiro | a3d8e05 | 2009-01-07 18:08:19 -0800 | [diff] [blame] | 330 | static inline unsigned long |
Hugh Dickins | 4d7dcca | 2012-05-29 15:07:08 -0700 | [diff] [blame] | 331 | mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru) |
KOSAKI Motohiro | a3d8e05 | 2009-01-07 18:08:19 -0800 | [diff] [blame] | 332 | { |
| 333 | return 0; |
| 334 | } |
| 335 | |
Hugh Dickins | fa9add6 | 2012-05-29 15:07:09 -0700 | [diff] [blame] | 336 | static inline void |
| 337 | mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru, |
| 338 | int increment) |
KOSAKI Motohiro | 3e2f41f | 2009-01-07 18:08:20 -0800 | [diff] [blame] | 339 | { |
KOSAKI Motohiro | 3e2f41f | 2009-01-07 18:08:20 -0800 | [diff] [blame] | 340 | } |
| 341 | |
Balbir Singh | e222432 | 2009-04-02 16:57:39 -0700 | [diff] [blame] | 342 | static inline void |
| 343 | mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p) |
| 344 | { |
| 345 | } |
| 346 | |
KAMEZAWA Hiroyuki | 89c06bd | 2012-03-21 16:34:25 -0700 | [diff] [blame] | 347 | static inline void mem_cgroup_begin_update_page_stat(struct page *page, |
| 348 | bool *locked, unsigned long *flags) |
| 349 | { |
| 350 | } |
| 351 | |
| 352 | static inline void mem_cgroup_end_update_page_stat(struct page *page, |
| 353 | bool *locked, unsigned long *flags) |
| 354 | { |
| 355 | } |
| 356 | |
Greg Thelen | 2a7106f | 2011-01-13 15:47:37 -0800 | [diff] [blame] | 357 | static inline void mem_cgroup_inc_page_stat(struct page *page, |
| 358 | enum mem_cgroup_page_stat_item idx) |
| 359 | { |
| 360 | } |
| 361 | |
| 362 | static inline void mem_cgroup_dec_page_stat(struct page *page, |
| 363 | enum mem_cgroup_page_stat_item idx) |
Balbir Singh | d69b042 | 2009-06-17 16:26:34 -0700 | [diff] [blame] | 364 | { |
| 365 | } |
| 366 | |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 367 | static inline |
| 368 | unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order, |
Ying Han | 0ae5e89 | 2011-05-26 16:25:25 -0700 | [diff] [blame] | 369 | gfp_t gfp_mask, |
| 370 | unsigned long *total_scanned) |
Balbir Singh | 4e41695 | 2009-09-23 15:56:39 -0700 | [diff] [blame] | 371 | { |
| 372 | return 0; |
| 373 | } |
| 374 | |
KAMEZAWA Hiroyuki | e94c8a9 | 2012-01-12 17:18:20 -0800 | [diff] [blame] | 375 | static inline void mem_cgroup_split_huge_fixup(struct page *head) |
KAMEZAWA Hiroyuki | ca3e021 | 2011-01-20 14:44:24 -0800 | [diff] [blame] | 376 | { |
| 377 | } |
| 378 | |
Ying Han | 456f998 | 2011-05-26 16:25:38 -0700 | [diff] [blame] | 379 | static inline |
| 380 | void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx) |
| 381 | { |
| 382 | } |
KAMEZAWA Hiroyuki | ab936cb | 2012-01-12 17:17:44 -0800 | [diff] [blame] | 383 | static inline void mem_cgroup_replace_page_cache(struct page *oldpage, |
| 384 | struct page *newpage) |
| 385 | { |
| 386 | } |
Andrew Morton | c255a45 | 2012-07-31 16:43:02 -0700 | [diff] [blame] | 387 | #endif /* CONFIG_MEMCG */ |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 388 | |
Andrew Morton | c255a45 | 2012-07-31 16:43:02 -0700 | [diff] [blame] | 389 | #if !defined(CONFIG_MEMCG) || !defined(CONFIG_DEBUG_VM) |
Daisuke Nishimura | f212ad7 | 2011-03-23 16:42:25 -0700 | [diff] [blame] | 390 | static inline bool |
| 391 | mem_cgroup_bad_page_check(struct page *page) |
| 392 | { |
| 393 | return false; |
| 394 | } |
| 395 | |
| 396 | static inline void |
| 397 | mem_cgroup_print_bad_page(struct page *page) |
| 398 | { |
| 399 | } |
| 400 | #endif |
| 401 | |
Glauber Costa | e1aab16 | 2011-12-11 21:47:03 +0000 | [diff] [blame] | 402 | enum { |
| 403 | UNDER_LIMIT, |
| 404 | SOFT_LIMIT, |
| 405 | OVER_LIMIT, |
| 406 | }; |
| 407 | |
| 408 | struct sock; |
David Rientjes | cd59085 | 2012-10-10 15:54:08 -0700 | [diff] [blame] | 409 | #if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM) |
Glauber Costa | e1aab16 | 2011-12-11 21:47:03 +0000 | [diff] [blame] | 410 | void sock_update_memcg(struct sock *sk); |
| 411 | void sock_release_memcg(struct sock *sk); |
| 412 | #else |
| 413 | static inline void sock_update_memcg(struct sock *sk) |
| 414 | { |
| 415 | } |
| 416 | static inline void sock_release_memcg(struct sock *sk) |
| 417 | { |
| 418 | } |
David Rientjes | cd59085 | 2012-10-10 15:54:08 -0700 | [diff] [blame] | 419 | #endif /* CONFIG_INET && CONFIG_MEMCG_KMEM */ |
Glauber Costa | 7ae1e1d | 2012-12-18 14:21:56 -0800 | [diff] [blame] | 420 | |
| 421 | #ifdef CONFIG_MEMCG_KMEM |
Glauber Costa | a8964b9 | 2012-12-18 14:22:09 -0800 | [diff] [blame] | 422 | extern struct static_key memcg_kmem_enabled_key; |
Glauber Costa | 749c541 | 2012-12-18 14:23:01 -0800 | [diff] [blame] | 423 | |
| 424 | extern int memcg_limited_groups_array_size; |
Glauber Costa | ebe945c | 2012-12-18 14:23:10 -0800 | [diff] [blame] | 425 | |
| 426 | /* |
| 427 | * Helper macro to loop through all memcg-specific caches. Callers must still |
| 428 | * check if the cache is valid (it is either valid or NULL). |
| 429 | * the slab_mutex must be held when looping through those caches |
| 430 | */ |
Glauber Costa | 749c541 | 2012-12-18 14:23:01 -0800 | [diff] [blame] | 431 | #define for_each_memcg_cache_index(_idx) \ |
Glauber Costa | 91c777d | 2013-02-04 14:28:49 -0800 | [diff] [blame^] | 432 | for ((_idx) = 0; (_idx) < memcg_limited_groups_array_size; (_idx)++) |
Glauber Costa | 749c541 | 2012-12-18 14:23:01 -0800 | [diff] [blame] | 433 | |
Glauber Costa | 7ae1e1d | 2012-12-18 14:21:56 -0800 | [diff] [blame] | 434 | static inline bool memcg_kmem_enabled(void) |
| 435 | { |
Glauber Costa | a8964b9 | 2012-12-18 14:22:09 -0800 | [diff] [blame] | 436 | return static_key_false(&memcg_kmem_enabled_key); |
Glauber Costa | 7ae1e1d | 2012-12-18 14:21:56 -0800 | [diff] [blame] | 437 | } |
| 438 | |
| 439 | /* |
| 440 | * In general, we'll do everything in our power to not incur in any overhead |
| 441 | * for non-memcg users for the kmem functions. Not even a function call, if we |
| 442 | * can avoid it. |
| 443 | * |
| 444 | * Therefore, we'll inline all those functions so that in the best case, we'll |
| 445 | * see that kmemcg is off for everybody and proceed quickly. If it is on, |
| 446 | * we'll still do most of the flag checking inline. We check a lot of |
| 447 | * conditions, but because they are pretty simple, they are expected to be |
| 448 | * fast. |
| 449 | */ |
| 450 | bool __memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **memcg, |
| 451 | int order); |
| 452 | void __memcg_kmem_commit_charge(struct page *page, |
| 453 | struct mem_cgroup *memcg, int order); |
| 454 | void __memcg_kmem_uncharge_pages(struct page *page, int order); |
| 455 | |
Glauber Costa | 2633d7a | 2012-12-18 14:22:34 -0800 | [diff] [blame] | 456 | int memcg_cache_id(struct mem_cgroup *memcg); |
Glauber Costa | 943a451 | 2012-12-18 14:23:03 -0800 | [diff] [blame] | 457 | int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s, |
| 458 | struct kmem_cache *root_cache); |
Glauber Costa | 2633d7a | 2012-12-18 14:22:34 -0800 | [diff] [blame] | 459 | void memcg_release_cache(struct kmem_cache *cachep); |
| 460 | void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep); |
| 461 | |
Glauber Costa | 55007d8 | 2012-12-18 14:22:38 -0800 | [diff] [blame] | 462 | int memcg_update_cache_size(struct kmem_cache *s, int num_groups); |
| 463 | void memcg_update_array_size(int num_groups); |
Glauber Costa | d7f25f8 | 2012-12-18 14:22:40 -0800 | [diff] [blame] | 464 | |
| 465 | struct kmem_cache * |
| 466 | __memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp); |
| 467 | |
Glauber Costa | 1f458cb | 2012-12-18 14:22:50 -0800 | [diff] [blame] | 468 | void mem_cgroup_destroy_cache(struct kmem_cache *cachep); |
Glauber Costa | 7cf2798 | 2012-12-18 14:22:55 -0800 | [diff] [blame] | 469 | void kmem_cache_destroy_memcg_children(struct kmem_cache *s); |
Glauber Costa | 1f458cb | 2012-12-18 14:22:50 -0800 | [diff] [blame] | 470 | |
Glauber Costa | 7ae1e1d | 2012-12-18 14:21:56 -0800 | [diff] [blame] | 471 | /** |
| 472 | * memcg_kmem_newpage_charge: verify if a new kmem allocation is allowed. |
| 473 | * @gfp: the gfp allocation flags. |
| 474 | * @memcg: a pointer to the memcg this was charged against. |
| 475 | * @order: allocation order. |
| 476 | * |
| 477 | * returns true if the memcg where the current task belongs can hold this |
| 478 | * allocation. |
| 479 | * |
| 480 | * We return true automatically if this allocation is not to be accounted to |
| 481 | * any memcg. |
| 482 | */ |
| 483 | static inline bool |
| 484 | memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **memcg, int order) |
| 485 | { |
| 486 | if (!memcg_kmem_enabled()) |
| 487 | return true; |
| 488 | |
| 489 | /* |
| 490 | * __GFP_NOFAIL allocations will move on even if charging is not |
| 491 | * possible. Therefore we don't even try, and have this allocation |
| 492 | * unaccounted. We could in theory charge it with |
| 493 | * res_counter_charge_nofail, but we hope those allocations are rare, |
| 494 | * and won't be worth the trouble. |
| 495 | */ |
| 496 | if (!(gfp & __GFP_KMEMCG) || (gfp & __GFP_NOFAIL)) |
| 497 | return true; |
| 498 | if (in_interrupt() || (!current->mm) || (current->flags & PF_KTHREAD)) |
| 499 | return true; |
| 500 | |
| 501 | /* If the test is dying, just let it go. */ |
| 502 | if (unlikely(fatal_signal_pending(current))) |
| 503 | return true; |
| 504 | |
| 505 | return __memcg_kmem_newpage_charge(gfp, memcg, order); |
| 506 | } |
| 507 | |
| 508 | /** |
| 509 | * memcg_kmem_uncharge_pages: uncharge pages from memcg |
| 510 | * @page: pointer to struct page being freed |
| 511 | * @order: allocation order. |
| 512 | * |
| 513 | * there is no need to specify memcg here, since it is embedded in page_cgroup |
| 514 | */ |
| 515 | static inline void |
| 516 | memcg_kmem_uncharge_pages(struct page *page, int order) |
| 517 | { |
| 518 | if (memcg_kmem_enabled()) |
| 519 | __memcg_kmem_uncharge_pages(page, order); |
| 520 | } |
| 521 | |
| 522 | /** |
| 523 | * memcg_kmem_commit_charge: embeds correct memcg in a page |
| 524 | * @page: pointer to struct page recently allocated |
| 525 | * @memcg: the memcg structure we charged against |
| 526 | * @order: allocation order. |
| 527 | * |
| 528 | * Needs to be called after memcg_kmem_newpage_charge, regardless of success or |
| 529 | * failure of the allocation. if @page is NULL, this function will revert the |
| 530 | * charges. Otherwise, it will commit the memcg given by @memcg to the |
| 531 | * corresponding page_cgroup. |
| 532 | */ |
| 533 | static inline void |
| 534 | memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg, int order) |
| 535 | { |
| 536 | if (memcg_kmem_enabled() && memcg) |
| 537 | __memcg_kmem_commit_charge(page, memcg, order); |
| 538 | } |
| 539 | |
Glauber Costa | d7f25f8 | 2012-12-18 14:22:40 -0800 | [diff] [blame] | 540 | /** |
| 541 | * memcg_kmem_get_cache: selects the correct per-memcg cache for allocation |
| 542 | * @cachep: the original global kmem cache |
| 543 | * @gfp: allocation flags. |
| 544 | * |
| 545 | * This function assumes that the task allocating, which determines the memcg |
| 546 | * in the page allocator, belongs to the same cgroup throughout the whole |
| 547 | * process. Misacounting can happen if the task calls memcg_kmem_get_cache() |
| 548 | * while belonging to a cgroup, and later on changes. This is considered |
| 549 | * acceptable, and should only happen upon task migration. |
| 550 | * |
| 551 | * Before the cache is created by the memcg core, there is also a possible |
| 552 | * imbalance: the task belongs to a memcg, but the cache being allocated from |
| 553 | * is the global cache, since the child cache is not yet guaranteed to be |
| 554 | * ready. This case is also fine, since in this case the GFP_KMEMCG will not be |
| 555 | * passed and the page allocator will not attempt any cgroup accounting. |
| 556 | */ |
| 557 | static __always_inline struct kmem_cache * |
| 558 | memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp) |
| 559 | { |
| 560 | if (!memcg_kmem_enabled()) |
| 561 | return cachep; |
| 562 | if (gfp & __GFP_NOFAIL) |
| 563 | return cachep; |
| 564 | if (in_interrupt() || (!current->mm) || (current->flags & PF_KTHREAD)) |
| 565 | return cachep; |
| 566 | if (unlikely(fatal_signal_pending(current))) |
| 567 | return cachep; |
| 568 | |
| 569 | return __memcg_kmem_get_cache(cachep, gfp); |
| 570 | } |
Glauber Costa | 7ae1e1d | 2012-12-18 14:21:56 -0800 | [diff] [blame] | 571 | #else |
Glauber Costa | 749c541 | 2012-12-18 14:23:01 -0800 | [diff] [blame] | 572 | #define for_each_memcg_cache_index(_idx) \ |
| 573 | for (; NULL; ) |
| 574 | |
Glauber Costa | b9ce5ef | 2012-12-18 14:22:46 -0800 | [diff] [blame] | 575 | static inline bool memcg_kmem_enabled(void) |
| 576 | { |
| 577 | return false; |
| 578 | } |
| 579 | |
Glauber Costa | 7ae1e1d | 2012-12-18 14:21:56 -0800 | [diff] [blame] | 580 | static inline bool |
| 581 | memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **memcg, int order) |
| 582 | { |
| 583 | return true; |
| 584 | } |
| 585 | |
| 586 | static inline void memcg_kmem_uncharge_pages(struct page *page, int order) |
| 587 | { |
| 588 | } |
| 589 | |
| 590 | static inline void |
| 591 | memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg, int order) |
| 592 | { |
| 593 | } |
Glauber Costa | 2633d7a | 2012-12-18 14:22:34 -0800 | [diff] [blame] | 594 | |
| 595 | static inline int memcg_cache_id(struct mem_cgroup *memcg) |
| 596 | { |
| 597 | return -1; |
| 598 | } |
| 599 | |
Glauber Costa | 943a451 | 2012-12-18 14:23:03 -0800 | [diff] [blame] | 600 | static inline int |
| 601 | memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s, |
| 602 | struct kmem_cache *root_cache) |
Glauber Costa | 2633d7a | 2012-12-18 14:22:34 -0800 | [diff] [blame] | 603 | { |
| 604 | return 0; |
| 605 | } |
| 606 | |
| 607 | static inline void memcg_release_cache(struct kmem_cache *cachep) |
| 608 | { |
| 609 | } |
| 610 | |
| 611 | static inline void memcg_cache_list_add(struct mem_cgroup *memcg, |
| 612 | struct kmem_cache *s) |
| 613 | { |
| 614 | } |
Glauber Costa | d7f25f8 | 2012-12-18 14:22:40 -0800 | [diff] [blame] | 615 | |
| 616 | static inline struct kmem_cache * |
| 617 | memcg_kmem_get_cache(struct kmem_cache *cachep, gfp_t gfp) |
| 618 | { |
| 619 | return cachep; |
| 620 | } |
Glauber Costa | 7cf2798 | 2012-12-18 14:22:55 -0800 | [diff] [blame] | 621 | |
| 622 | static inline void kmem_cache_destroy_memcg_children(struct kmem_cache *s) |
| 623 | { |
| 624 | } |
Glauber Costa | 7ae1e1d | 2012-12-18 14:21:56 -0800 | [diff] [blame] | 625 | #endif /* CONFIG_MEMCG_KMEM */ |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 626 | #endif /* _LINUX_MEMCONTROL_H */ |
| 627 | |