Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1 | /* memcontrol.c - Memory Controller |
| 2 | * |
| 3 | * Copyright IBM Corporation, 2007 |
| 4 | * Author Balbir Singh <balbir@linux.vnet.ibm.com> |
| 5 | * |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 6 | * Copyright 2007 OpenVZ SWsoft Inc |
| 7 | * Author: Pavel Emelianov <xemul@openvz.org> |
| 8 | * |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 9 | * This program is free software; you can redistribute it and/or modify |
| 10 | * it under the terms of the GNU General Public License as published by |
| 11 | * the Free Software Foundation; either version 2 of the License, or |
| 12 | * (at your option) any later version. |
| 13 | * |
| 14 | * This program is distributed in the hope that it will be useful, |
| 15 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 16 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 17 | * GNU General Public License for more details. |
| 18 | */ |
| 19 | |
| 20 | #include <linux/res_counter.h> |
| 21 | #include <linux/memcontrol.h> |
| 22 | #include <linux/cgroup.h> |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 23 | #include <linux/mm.h> |
KAMEZAWA Hiroyuki | d52aa41 | 2008-02-07 00:14:24 -0800 | [diff] [blame] | 24 | #include <linux/smp.h> |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 25 | #include <linux/page-flags.h> |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 26 | #include <linux/backing-dev.h> |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 27 | #include <linux/bit_spinlock.h> |
| 28 | #include <linux/rcupdate.h> |
Balbir Singh | b6ac57d | 2008-04-29 01:00:19 -0700 | [diff] [blame] | 29 | #include <linux/slab.h> |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 30 | #include <linux/swap.h> |
| 31 | #include <linux/spinlock.h> |
| 32 | #include <linux/fs.h> |
KAMEZAWA Hiroyuki | d2ceb9b | 2008-02-07 00:14:25 -0800 | [diff] [blame] | 33 | #include <linux/seq_file.h> |
KAMEZAWA Hiroyuki | 3332794 | 2008-04-29 01:00:24 -0700 | [diff] [blame] | 34 | #include <linux/vmalloc.h> |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 35 | #include <linux/mm_inline.h> |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 36 | #include <linux/page_cgroup.h> |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 37 | |
Balbir Singh | 8697d33 | 2008-02-07 00:13:59 -0800 | [diff] [blame] | 38 | #include <asm/uaccess.h> |
| 39 | |
KAMEZAWA Hiroyuki | a181b0e | 2008-07-25 01:47:08 -0700 | [diff] [blame] | 40 | struct cgroup_subsys mem_cgroup_subsys __read_mostly; |
KAMEZAWA Hiroyuki | a181b0e | 2008-07-25 01:47:08 -0700 | [diff] [blame] | 41 | #define MEM_CGROUP_RECLAIM_RETRIES 5 |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 42 | |
| 43 | /* |
KAMEZAWA Hiroyuki | d52aa41 | 2008-02-07 00:14:24 -0800 | [diff] [blame] | 44 | * Statistics for memory cgroup. |
| 45 | */ |
| 46 | enum mem_cgroup_stat_index { |
| 47 | /* |
| 48 | * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss. |
| 49 | */ |
| 50 | MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */ |
| 51 | MEM_CGROUP_STAT_RSS, /* # of pages charged as rss */ |
Balaji Rao | 55e462b | 2008-05-01 04:35:12 -0700 | [diff] [blame] | 52 | MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */ |
| 53 | MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */ |
KAMEZAWA Hiroyuki | d52aa41 | 2008-02-07 00:14:24 -0800 | [diff] [blame] | 54 | |
| 55 | MEM_CGROUP_STAT_NSTATS, |
| 56 | }; |
| 57 | |
| 58 | struct mem_cgroup_stat_cpu { |
| 59 | s64 count[MEM_CGROUP_STAT_NSTATS]; |
| 60 | } ____cacheline_aligned_in_smp; |
| 61 | |
| 62 | struct mem_cgroup_stat { |
| 63 | struct mem_cgroup_stat_cpu cpustat[NR_CPUS]; |
| 64 | }; |
| 65 | |
| 66 | /* |
| 67 | * For accounting under irq disable, no need for increment preempt count. |
| 68 | */ |
KAMEZAWA Hiroyuki | addb9ef | 2008-10-18 20:28:10 -0700 | [diff] [blame] | 69 | static inline void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat_cpu *stat, |
KAMEZAWA Hiroyuki | d52aa41 | 2008-02-07 00:14:24 -0800 | [diff] [blame] | 70 | enum mem_cgroup_stat_index idx, int val) |
| 71 | { |
KAMEZAWA Hiroyuki | addb9ef | 2008-10-18 20:28:10 -0700 | [diff] [blame] | 72 | stat->count[idx] += val; |
KAMEZAWA Hiroyuki | d52aa41 | 2008-02-07 00:14:24 -0800 | [diff] [blame] | 73 | } |
| 74 | |
| 75 | static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat, |
| 76 | enum mem_cgroup_stat_index idx) |
| 77 | { |
| 78 | int cpu; |
| 79 | s64 ret = 0; |
| 80 | for_each_possible_cpu(cpu) |
| 81 | ret += stat->cpustat[cpu].count[idx]; |
| 82 | return ret; |
| 83 | } |
| 84 | |
| 85 | /* |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 86 | * per-zone information in memory controller. |
| 87 | */ |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 88 | struct mem_cgroup_per_zone { |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 89 | /* |
| 90 | * spin_lock to protect the per cgroup LRU |
| 91 | */ |
| 92 | spinlock_t lru_lock; |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 93 | struct list_head lists[NR_LRU_LISTS]; |
| 94 | unsigned long count[NR_LRU_LISTS]; |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 95 | }; |
| 96 | /* Macro for accessing counter */ |
| 97 | #define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)]) |
| 98 | |
| 99 | struct mem_cgroup_per_node { |
| 100 | struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES]; |
| 101 | }; |
| 102 | |
| 103 | struct mem_cgroup_lru_info { |
| 104 | struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES]; |
| 105 | }; |
| 106 | |
| 107 | /* |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 108 | * The memory controller data structure. The memory controller controls both |
| 109 | * page cache and RSS per cgroup. We would eventually like to provide |
| 110 | * statistics based on the statistics developed by Rik Van Riel for clock-pro, |
| 111 | * to help the administrator determine what knobs to tune. |
| 112 | * |
| 113 | * TODO: Add a water mark for the memory controller. Reclaim will begin when |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 114 | * we hit the water mark. May be even add a low water mark, such that |
| 115 | * no reclaim occurs from a cgroup at it's low water mark, this is |
| 116 | * a feature that will be implemented much later in the future. |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 117 | */ |
| 118 | struct mem_cgroup { |
| 119 | struct cgroup_subsys_state css; |
| 120 | /* |
| 121 | * the counter to account for memory usage |
| 122 | */ |
| 123 | struct res_counter res; |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 124 | /* |
| 125 | * Per cgroup active and inactive list, similar to the |
| 126 | * per zone LRU lists. |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 127 | */ |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 128 | struct mem_cgroup_lru_info info; |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 129 | |
KAMEZAWA Hiroyuki | 6c48a1d | 2008-02-07 00:14:34 -0800 | [diff] [blame] | 130 | int prev_priority; /* for recording reclaim priority */ |
KAMEZAWA Hiroyuki | d52aa41 | 2008-02-07 00:14:24 -0800 | [diff] [blame] | 131 | /* |
| 132 | * statistics. |
| 133 | */ |
| 134 | struct mem_cgroup_stat stat; |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 135 | }; |
Hugh Dickins | 8869b8f | 2008-03-04 14:29:09 -0800 | [diff] [blame] | 136 | static struct mem_cgroup init_mem_cgroup; |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 137 | |
KAMEZAWA Hiroyuki | 217bc31 | 2008-02-07 00:14:17 -0800 | [diff] [blame] | 138 | enum charge_type { |
| 139 | MEM_CGROUP_CHARGE_TYPE_CACHE = 0, |
| 140 | MEM_CGROUP_CHARGE_TYPE_MAPPED, |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 141 | MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */ |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 142 | MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */ |
| 143 | NR_CHARGE_TYPE, |
| 144 | }; |
| 145 | |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 146 | /* only for here (for easy reading.) */ |
| 147 | #define PCGF_CACHE (1UL << PCG_CACHE) |
| 148 | #define PCGF_USED (1UL << PCG_USED) |
| 149 | #define PCGF_ACTIVE (1UL << PCG_ACTIVE) |
| 150 | #define PCGF_LOCK (1UL << PCG_LOCK) |
| 151 | #define PCGF_FILE (1UL << PCG_FILE) |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 152 | static const unsigned long |
| 153 | pcg_default_flags[NR_CHARGE_TYPE] = { |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 154 | PCGF_CACHE | PCGF_FILE | PCGF_USED | PCGF_LOCK, /* File Cache */ |
| 155 | PCGF_ACTIVE | PCGF_USED | PCGF_LOCK, /* Anon */ |
| 156 | PCGF_ACTIVE | PCGF_CACHE | PCGF_USED | PCGF_LOCK, /* Shmem */ |
| 157 | 0, /* FORCE */ |
KAMEZAWA Hiroyuki | 217bc31 | 2008-02-07 00:14:17 -0800 | [diff] [blame] | 158 | }; |
| 159 | |
KAMEZAWA Hiroyuki | d52aa41 | 2008-02-07 00:14:24 -0800 | [diff] [blame] | 160 | /* |
| 161 | * Always modified under lru lock. Then, not necessary to preempt_disable() |
| 162 | */ |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 163 | static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, |
| 164 | struct page_cgroup *pc, |
| 165 | bool charge) |
KAMEZAWA Hiroyuki | d52aa41 | 2008-02-07 00:14:24 -0800 | [diff] [blame] | 166 | { |
| 167 | int val = (charge)? 1 : -1; |
| 168 | struct mem_cgroup_stat *stat = &mem->stat; |
KAMEZAWA Hiroyuki | addb9ef | 2008-10-18 20:28:10 -0700 | [diff] [blame] | 169 | struct mem_cgroup_stat_cpu *cpustat; |
KAMEZAWA Hiroyuki | d52aa41 | 2008-02-07 00:14:24 -0800 | [diff] [blame] | 170 | |
Hugh Dickins | 8869b8f | 2008-03-04 14:29:09 -0800 | [diff] [blame] | 171 | VM_BUG_ON(!irqs_disabled()); |
KAMEZAWA Hiroyuki | addb9ef | 2008-10-18 20:28:10 -0700 | [diff] [blame] | 172 | |
| 173 | cpustat = &stat->cpustat[smp_processor_id()]; |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 174 | if (PageCgroupCache(pc)) |
KAMEZAWA Hiroyuki | addb9ef | 2008-10-18 20:28:10 -0700 | [diff] [blame] | 175 | __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_CACHE, val); |
KAMEZAWA Hiroyuki | d52aa41 | 2008-02-07 00:14:24 -0800 | [diff] [blame] | 176 | else |
KAMEZAWA Hiroyuki | addb9ef | 2008-10-18 20:28:10 -0700 | [diff] [blame] | 177 | __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_RSS, val); |
Balaji Rao | 55e462b | 2008-05-01 04:35:12 -0700 | [diff] [blame] | 178 | |
| 179 | if (charge) |
KAMEZAWA Hiroyuki | addb9ef | 2008-10-18 20:28:10 -0700 | [diff] [blame] | 180 | __mem_cgroup_stat_add_safe(cpustat, |
Balaji Rao | 55e462b | 2008-05-01 04:35:12 -0700 | [diff] [blame] | 181 | MEM_CGROUP_STAT_PGPGIN_COUNT, 1); |
| 182 | else |
KAMEZAWA Hiroyuki | addb9ef | 2008-10-18 20:28:10 -0700 | [diff] [blame] | 183 | __mem_cgroup_stat_add_safe(cpustat, |
Balaji Rao | 55e462b | 2008-05-01 04:35:12 -0700 | [diff] [blame] | 184 | MEM_CGROUP_STAT_PGPGOUT_COUNT, 1); |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 185 | } |
KAMEZAWA Hiroyuki | d52aa41 | 2008-02-07 00:14:24 -0800 | [diff] [blame] | 186 | |
Hugh Dickins | d5b69e3 | 2008-03-04 14:29:10 -0800 | [diff] [blame] | 187 | static struct mem_cgroup_per_zone * |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 188 | mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid) |
| 189 | { |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 190 | return &mem->info.nodeinfo[nid]->zoneinfo[zid]; |
| 191 | } |
| 192 | |
Hugh Dickins | d5b69e3 | 2008-03-04 14:29:10 -0800 | [diff] [blame] | 193 | static struct mem_cgroup_per_zone * |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 194 | page_cgroup_zoneinfo(struct page_cgroup *pc) |
| 195 | { |
| 196 | struct mem_cgroup *mem = pc->mem_cgroup; |
| 197 | int nid = page_cgroup_nid(pc); |
| 198 | int zid = page_cgroup_zid(pc); |
| 199 | |
| 200 | return mem_cgroup_zoneinfo(mem, nid, zid); |
| 201 | } |
| 202 | |
| 203 | static unsigned long mem_cgroup_get_all_zonestat(struct mem_cgroup *mem, |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 204 | enum lru_list idx) |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 205 | { |
| 206 | int nid, zid; |
| 207 | struct mem_cgroup_per_zone *mz; |
| 208 | u64 total = 0; |
| 209 | |
| 210 | for_each_online_node(nid) |
| 211 | for (zid = 0; zid < MAX_NR_ZONES; zid++) { |
| 212 | mz = mem_cgroup_zoneinfo(mem, nid, zid); |
| 213 | total += MEM_CGROUP_ZSTAT(mz, idx); |
| 214 | } |
| 215 | return total; |
KAMEZAWA Hiroyuki | d52aa41 | 2008-02-07 00:14:24 -0800 | [diff] [blame] | 216 | } |
| 217 | |
Hugh Dickins | d5b69e3 | 2008-03-04 14:29:10 -0800 | [diff] [blame] | 218 | static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont) |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 219 | { |
| 220 | return container_of(cgroup_subsys_state(cont, |
| 221 | mem_cgroup_subsys_id), struct mem_cgroup, |
| 222 | css); |
| 223 | } |
| 224 | |
Balbir Singh | cf475ad | 2008-04-29 01:00:16 -0700 | [diff] [blame] | 225 | struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p) |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 226 | { |
Balbir Singh | 31a78f2 | 2008-09-28 23:09:31 +0100 | [diff] [blame] | 227 | /* |
| 228 | * mm_update_next_owner() may clear mm->owner to NULL |
| 229 | * if it races with swapoff, page migration, etc. |
| 230 | * So this can be called with p == NULL. |
| 231 | */ |
| 232 | if (unlikely(!p)) |
| 233 | return NULL; |
| 234 | |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 235 | return container_of(task_subsys_state(p, mem_cgroup_subsys_id), |
| 236 | struct mem_cgroup, css); |
| 237 | } |
| 238 | |
KAMEZAWA Hiroyuki | 3eae90c | 2008-04-29 01:00:22 -0700 | [diff] [blame] | 239 | static void __mem_cgroup_remove_list(struct mem_cgroup_per_zone *mz, |
| 240 | struct page_cgroup *pc) |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 241 | { |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 242 | int lru = LRU_BASE; |
| 243 | |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 244 | if (PageCgroupUnevictable(pc)) |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 245 | lru = LRU_UNEVICTABLE; |
| 246 | else { |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 247 | if (PageCgroupActive(pc)) |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 248 | lru += LRU_ACTIVE; |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 249 | if (PageCgroupFile(pc)) |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 250 | lru += LRU_FILE; |
| 251 | } |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 252 | |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 253 | MEM_CGROUP_ZSTAT(mz, lru) -= 1; |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 254 | |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 255 | mem_cgroup_charge_statistics(pc->mem_cgroup, pc, false); |
KAMEZAWA Hiroyuki | 508b7be | 2008-07-25 01:47:09 -0700 | [diff] [blame] | 256 | list_del(&pc->lru); |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 257 | } |
| 258 | |
KAMEZAWA Hiroyuki | 3eae90c | 2008-04-29 01:00:22 -0700 | [diff] [blame] | 259 | static void __mem_cgroup_add_list(struct mem_cgroup_per_zone *mz, |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 260 | struct page_cgroup *pc, bool hot) |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 261 | { |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 262 | int lru = LRU_BASE; |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 263 | |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 264 | if (PageCgroupUnevictable(pc)) |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 265 | lru = LRU_UNEVICTABLE; |
| 266 | else { |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 267 | if (PageCgroupActive(pc)) |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 268 | lru += LRU_ACTIVE; |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 269 | if (PageCgroupFile(pc)) |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 270 | lru += LRU_FILE; |
| 271 | } |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 272 | |
| 273 | MEM_CGROUP_ZSTAT(mz, lru) += 1; |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 274 | if (hot) |
| 275 | list_add(&pc->lru, &mz->lists[lru]); |
| 276 | else |
| 277 | list_add_tail(&pc->lru, &mz->lists[lru]); |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 278 | |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 279 | mem_cgroup_charge_statistics(pc->mem_cgroup, pc, true); |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 280 | } |
| 281 | |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 282 | static void __mem_cgroup_move_lists(struct page_cgroup *pc, enum lru_list lru) |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 283 | { |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 284 | struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc); |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 285 | int active = PageCgroupActive(pc); |
| 286 | int file = PageCgroupFile(pc); |
| 287 | int unevictable = PageCgroupUnevictable(pc); |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 288 | enum lru_list from = unevictable ? LRU_UNEVICTABLE : |
| 289 | (LRU_FILE * !!file + !!active); |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 290 | |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 291 | if (lru == from) |
| 292 | return; |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 293 | |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 294 | MEM_CGROUP_ZSTAT(mz, from) -= 1; |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 295 | /* |
| 296 | * However this is done under mz->lru_lock, another flags, which |
| 297 | * are not related to LRU, will be modified from out-of-lock. |
| 298 | * We have to use atomic set/clear flags. |
| 299 | */ |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 300 | if (is_unevictable_lru(lru)) { |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 301 | ClearPageCgroupActive(pc); |
| 302 | SetPageCgroupUnevictable(pc); |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 303 | } else { |
| 304 | if (is_active_lru(lru)) |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 305 | SetPageCgroupActive(pc); |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 306 | else |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 307 | ClearPageCgroupActive(pc); |
| 308 | ClearPageCgroupUnevictable(pc); |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 309 | } |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 310 | |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 311 | MEM_CGROUP_ZSTAT(mz, lru) += 1; |
| 312 | list_move(&pc->lru, &mz->lists[lru]); |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 313 | } |
| 314 | |
David Rientjes | 4c4a221 | 2008-02-07 00:14:06 -0800 | [diff] [blame] | 315 | int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem) |
| 316 | { |
| 317 | int ret; |
| 318 | |
| 319 | task_lock(task); |
Hugh Dickins | bd845e3 | 2008-03-04 14:29:01 -0800 | [diff] [blame] | 320 | ret = task->mm && mm_match_cgroup(task->mm, mem); |
David Rientjes | 4c4a221 | 2008-02-07 00:14:06 -0800 | [diff] [blame] | 321 | task_unlock(task); |
| 322 | return ret; |
| 323 | } |
| 324 | |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 325 | /* |
| 326 | * This routine assumes that the appropriate zone's lru lock is already held |
| 327 | */ |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 328 | void mem_cgroup_move_lists(struct page *page, enum lru_list lru) |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 329 | { |
Hugh Dickins | 427d541 | 2008-03-04 14:29:03 -0800 | [diff] [blame] | 330 | struct page_cgroup *pc; |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 331 | struct mem_cgroup_per_zone *mz; |
| 332 | unsigned long flags; |
| 333 | |
Li Zefan | cede86a | 2008-07-25 01:47:18 -0700 | [diff] [blame] | 334 | if (mem_cgroup_subsys.disabled) |
| 335 | return; |
| 336 | |
Hugh Dickins | 2680eed | 2008-03-04 14:29:13 -0800 | [diff] [blame] | 337 | /* |
| 338 | * We cannot lock_page_cgroup while holding zone's lru_lock, |
| 339 | * because other holders of lock_page_cgroup can be interrupted |
| 340 | * with an attempt to rotate_reclaimable_page. But we cannot |
| 341 | * safely get to page_cgroup without it, so just try_lock it: |
| 342 | * mem_cgroup_isolate_pages allows for page left on wrong list. |
| 343 | */ |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 344 | pc = lookup_page_cgroup(page); |
| 345 | if (!trylock_page_cgroup(pc)) |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 346 | return; |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 347 | if (pc && PageCgroupUsed(pc)) { |
Hugh Dickins | 2680eed | 2008-03-04 14:29:13 -0800 | [diff] [blame] | 348 | mz = page_cgroup_zoneinfo(pc); |
Hugh Dickins | 2680eed | 2008-03-04 14:29:13 -0800 | [diff] [blame] | 349 | spin_lock_irqsave(&mz->lru_lock, flags); |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 350 | __mem_cgroup_move_lists(pc, lru); |
Hugh Dickins | 2680eed | 2008-03-04 14:29:13 -0800 | [diff] [blame] | 351 | spin_unlock_irqrestore(&mz->lru_lock, flags); |
Hirokazu Takahashi | 9b3c0a0 | 2008-03-04 14:29:15 -0800 | [diff] [blame] | 352 | } |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 353 | unlock_page_cgroup(pc); |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 354 | } |
| 355 | |
KAMEZAWA Hiroyuki | 58ae83d | 2008-02-07 00:14:32 -0800 | [diff] [blame] | 356 | /* |
| 357 | * Calculate mapped_ratio under memory controller. This will be used in |
| 358 | * vmscan.c for deteremining we have to reclaim mapped pages. |
| 359 | */ |
| 360 | int mem_cgroup_calc_mapped_ratio(struct mem_cgroup *mem) |
| 361 | { |
| 362 | long total, rss; |
| 363 | |
| 364 | /* |
| 365 | * usage is recorded in bytes. But, here, we assume the number of |
| 366 | * physical pages can be represented by "long" on any arch. |
| 367 | */ |
| 368 | total = (long) (mem->res.usage >> PAGE_SHIFT) + 1L; |
| 369 | rss = (long)mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS); |
| 370 | return (int)((rss * 100L) / total); |
| 371 | } |
Hugh Dickins | 8869b8f | 2008-03-04 14:29:09 -0800 | [diff] [blame] | 372 | |
KAMEZAWA Hiroyuki | 5932f36 | 2008-02-07 00:14:33 -0800 | [diff] [blame] | 373 | /* |
KAMEZAWA Hiroyuki | 6c48a1d | 2008-02-07 00:14:34 -0800 | [diff] [blame] | 374 | * prev_priority control...this will be used in memory reclaim path. |
| 375 | */ |
| 376 | int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem) |
| 377 | { |
| 378 | return mem->prev_priority; |
| 379 | } |
| 380 | |
| 381 | void mem_cgroup_note_reclaim_priority(struct mem_cgroup *mem, int priority) |
| 382 | { |
| 383 | if (priority < mem->prev_priority) |
| 384 | mem->prev_priority = priority; |
| 385 | } |
| 386 | |
| 387 | void mem_cgroup_record_reclaim_priority(struct mem_cgroup *mem, int priority) |
| 388 | { |
| 389 | mem->prev_priority = priority; |
| 390 | } |
| 391 | |
KAMEZAWA Hiroyuki | cc38108 | 2008-02-07 00:14:35 -0800 | [diff] [blame] | 392 | /* |
| 393 | * Calculate # of pages to be scanned in this priority/zone. |
| 394 | * See also vmscan.c |
| 395 | * |
| 396 | * priority starts from "DEF_PRIORITY" and decremented in each loop. |
| 397 | * (see include/linux/mmzone.h) |
| 398 | */ |
| 399 | |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 400 | long mem_cgroup_calc_reclaim(struct mem_cgroup *mem, struct zone *zone, |
| 401 | int priority, enum lru_list lru) |
KAMEZAWA Hiroyuki | cc38108 | 2008-02-07 00:14:35 -0800 | [diff] [blame] | 402 | { |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 403 | long nr_pages; |
KAMEZAWA Hiroyuki | cc38108 | 2008-02-07 00:14:35 -0800 | [diff] [blame] | 404 | int nid = zone->zone_pgdat->node_id; |
| 405 | int zid = zone_idx(zone); |
| 406 | struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid); |
| 407 | |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 408 | nr_pages = MEM_CGROUP_ZSTAT(mz, lru); |
KAMEZAWA Hiroyuki | cc38108 | 2008-02-07 00:14:35 -0800 | [diff] [blame] | 409 | |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 410 | return (nr_pages >> priority); |
KAMEZAWA Hiroyuki | cc38108 | 2008-02-07 00:14:35 -0800 | [diff] [blame] | 411 | } |
| 412 | |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 413 | unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan, |
| 414 | struct list_head *dst, |
| 415 | unsigned long *scanned, int order, |
| 416 | int mode, struct zone *z, |
| 417 | struct mem_cgroup *mem_cont, |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 418 | int active, int file) |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 419 | { |
| 420 | unsigned long nr_taken = 0; |
| 421 | struct page *page; |
| 422 | unsigned long scan; |
| 423 | LIST_HEAD(pc_list); |
| 424 | struct list_head *src; |
KAMEZAWA Hiroyuki | ff7283f | 2008-02-07 00:14:11 -0800 | [diff] [blame] | 425 | struct page_cgroup *pc, *tmp; |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 426 | int nid = z->zone_pgdat->node_id; |
| 427 | int zid = zone_idx(z); |
| 428 | struct mem_cgroup_per_zone *mz; |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 429 | int lru = LRU_FILE * !!file + !!active; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 430 | |
Balbir Singh | cf475ad | 2008-04-29 01:00:16 -0700 | [diff] [blame] | 431 | BUG_ON(!mem_cont); |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 432 | mz = mem_cgroup_zoneinfo(mem_cont, nid, zid); |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 433 | src = &mz->lists[lru]; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 434 | |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 435 | spin_lock(&mz->lru_lock); |
KAMEZAWA Hiroyuki | ff7283f | 2008-02-07 00:14:11 -0800 | [diff] [blame] | 436 | scan = 0; |
| 437 | list_for_each_entry_safe_reverse(pc, tmp, src, lru) { |
Hugh Dickins | 436c6541 | 2008-02-07 00:14:12 -0800 | [diff] [blame] | 438 | if (scan >= nr_to_scan) |
KAMEZAWA Hiroyuki | ff7283f | 2008-02-07 00:14:11 -0800 | [diff] [blame] | 439 | break; |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 440 | if (unlikely(!PageCgroupUsed(pc))) |
| 441 | continue; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 442 | page = pc->page; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 443 | |
Hugh Dickins | 436c6541 | 2008-02-07 00:14:12 -0800 | [diff] [blame] | 444 | if (unlikely(!PageLRU(page))) |
KAMEZAWA Hiroyuki | ff7283f | 2008-02-07 00:14:11 -0800 | [diff] [blame] | 445 | continue; |
KAMEZAWA Hiroyuki | ff7283f | 2008-02-07 00:14:11 -0800 | [diff] [blame] | 446 | |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 447 | /* |
| 448 | * TODO: play better with lumpy reclaim, grabbing anything. |
| 449 | */ |
Lee Schermerhorn | 894bc31 | 2008-10-18 20:26:39 -0700 | [diff] [blame] | 450 | if (PageUnevictable(page) || |
| 451 | (PageActive(page) && !active) || |
| 452 | (!PageActive(page) && active)) { |
| 453 | __mem_cgroup_move_lists(pc, page_lru(page)); |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 454 | continue; |
| 455 | } |
| 456 | |
Hugh Dickins | 436c6541 | 2008-02-07 00:14:12 -0800 | [diff] [blame] | 457 | scan++; |
| 458 | list_move(&pc->lru, &pc_list); |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 459 | |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 460 | if (__isolate_lru_page(page, mode, file) == 0) { |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 461 | list_move(&page->lru, dst); |
| 462 | nr_taken++; |
| 463 | } |
| 464 | } |
| 465 | |
| 466 | list_splice(&pc_list, src); |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 467 | spin_unlock(&mz->lru_lock); |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 468 | |
| 469 | *scanned = scan; |
| 470 | return nr_taken; |
| 471 | } |
| 472 | |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 473 | /* |
| 474 | * Unlike exported interface, "oom" parameter is added. if oom==true, |
| 475 | * oom-killer can be invoked. |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 476 | */ |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 477 | static int __mem_cgroup_try_charge(struct mm_struct *mm, |
| 478 | gfp_t gfp_mask, struct mem_cgroup **memcg, bool oom) |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 479 | { |
| 480 | struct mem_cgroup *mem; |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 481 | int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 482 | /* |
Hugh Dickins | 3be9127 | 2008-02-07 00:14:19 -0800 | [diff] [blame] | 483 | * We always charge the cgroup the mm_struct belongs to. |
| 484 | * The mm_struct's mem_cgroup changes on task migration if the |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 485 | * thread group leader migrates. It's possible that mm is not |
| 486 | * set, if so charge the init_mm (happens for pagecache usage). |
| 487 | */ |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 488 | if (likely(!*memcg)) { |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 489 | rcu_read_lock(); |
| 490 | mem = mem_cgroup_from_task(rcu_dereference(mm->owner)); |
Balbir Singh | 31a78f2 | 2008-09-28 23:09:31 +0100 | [diff] [blame] | 491 | if (unlikely(!mem)) { |
| 492 | rcu_read_unlock(); |
Balbir Singh | 31a78f2 | 2008-09-28 23:09:31 +0100 | [diff] [blame] | 493 | return 0; |
| 494 | } |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 495 | /* |
| 496 | * For every charge from the cgroup, increment reference count |
| 497 | */ |
| 498 | css_get(&mem->css); |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 499 | *memcg = mem; |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 500 | rcu_read_unlock(); |
| 501 | } else { |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 502 | mem = *memcg; |
| 503 | css_get(&mem->css); |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 504 | } |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 505 | |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 506 | |
KAMEZAWA Hiroyuki | addb9ef | 2008-10-18 20:28:10 -0700 | [diff] [blame] | 507 | while (unlikely(res_counter_charge(&mem->res, PAGE_SIZE))) { |
Hugh Dickins | 3be9127 | 2008-02-07 00:14:19 -0800 | [diff] [blame] | 508 | if (!(gfp_mask & __GFP_WAIT)) |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 509 | goto nomem; |
Balbir Singh | e1a1cd5 | 2008-02-07 00:14:02 -0800 | [diff] [blame] | 510 | |
| 511 | if (try_to_free_mem_cgroup_pages(mem, gfp_mask)) |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 512 | continue; |
| 513 | |
| 514 | /* |
Hugh Dickins | 8869b8f | 2008-03-04 14:29:09 -0800 | [diff] [blame] | 515 | * try_to_free_mem_cgroup_pages() might not give us a full |
| 516 | * picture of reclaim. Some pages are reclaimed and might be |
| 517 | * moved to swap cache or just unmapped from the cgroup. |
| 518 | * Check the limit again to see if the reclaim reduced the |
| 519 | * current usage of the cgroup before giving up |
| 520 | */ |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 521 | if (res_counter_check_under_limit(&mem->res)) |
| 522 | continue; |
Hugh Dickins | 3be9127 | 2008-02-07 00:14:19 -0800 | [diff] [blame] | 523 | |
| 524 | if (!nr_retries--) { |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 525 | if (oom) |
| 526 | mem_cgroup_out_of_memory(mem, gfp_mask); |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 527 | goto nomem; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 528 | } |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 529 | } |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 530 | return 0; |
| 531 | nomem: |
| 532 | css_put(&mem->css); |
| 533 | return -ENOMEM; |
| 534 | } |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 535 | |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 536 | /** |
| 537 | * mem_cgroup_try_charge - get charge of PAGE_SIZE. |
| 538 | * @mm: an mm_struct which is charged against. (when *memcg is NULL) |
| 539 | * @gfp_mask: gfp_mask for reclaim. |
| 540 | * @memcg: a pointer to memory cgroup which is charged against. |
| 541 | * |
| 542 | * charge against memory cgroup pointed by *memcg. if *memcg == NULL, estimated |
| 543 | * memory cgroup from @mm is got and stored in *memcg. |
| 544 | * |
| 545 | * Returns 0 if success. -ENOMEM at failure. |
| 546 | * This call can invoke OOM-Killer. |
| 547 | */ |
| 548 | |
| 549 | int mem_cgroup_try_charge(struct mm_struct *mm, |
| 550 | gfp_t mask, struct mem_cgroup **memcg) |
| 551 | { |
| 552 | return __mem_cgroup_try_charge(mm, mask, memcg, true); |
| 553 | } |
| 554 | |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 555 | /* |
| 556 | * commit a charge got by mem_cgroup_try_charge() and makes page_cgroup to be |
| 557 | * USED state. If already USED, uncharge and return. |
| 558 | */ |
| 559 | |
| 560 | static void __mem_cgroup_commit_charge(struct mem_cgroup *mem, |
| 561 | struct page_cgroup *pc, |
| 562 | enum charge_type ctype) |
| 563 | { |
| 564 | struct mem_cgroup_per_zone *mz; |
| 565 | unsigned long flags; |
| 566 | |
| 567 | /* try_charge() can return NULL to *memcg, taking care of it. */ |
| 568 | if (!mem) |
| 569 | return; |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 570 | |
| 571 | lock_page_cgroup(pc); |
| 572 | if (unlikely(PageCgroupUsed(pc))) { |
| 573 | unlock_page_cgroup(pc); |
| 574 | res_counter_uncharge(&mem->res, PAGE_SIZE); |
| 575 | css_put(&mem->css); |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 576 | return; |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 577 | } |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 578 | pc->mem_cgroup = mem; |
KAMEZAWA Hiroyuki | 508b7be | 2008-07-25 01:47:09 -0700 | [diff] [blame] | 579 | /* |
| 580 | * If a page is accounted as a page cache, insert to inactive list. |
| 581 | * If anon, insert to active list. |
| 582 | */ |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 583 | pc->flags = pcg_default_flags[ctype]; |
Hugh Dickins | 3be9127 | 2008-02-07 00:14:19 -0800 | [diff] [blame] | 584 | |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 585 | mz = page_cgroup_zoneinfo(pc); |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 586 | |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 587 | spin_lock_irqsave(&mz->lru_lock, flags); |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 588 | __mem_cgroup_add_list(mz, pc, true); |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 589 | spin_unlock_irqrestore(&mz->lru_lock, flags); |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 590 | unlock_page_cgroup(pc); |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 591 | } |
| 592 | |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 593 | /** |
| 594 | * mem_cgroup_move_account - move account of the page |
| 595 | * @pc: page_cgroup of the page. |
| 596 | * @from: mem_cgroup which the page is moved from. |
| 597 | * @to: mem_cgroup which the page is moved to. @from != @to. |
| 598 | * |
| 599 | * The caller must confirm following. |
| 600 | * 1. disable irq. |
| 601 | * 2. lru_lock of old mem_cgroup(@from) should be held. |
| 602 | * |
| 603 | * returns 0 at success, |
| 604 | * returns -EBUSY when lock is busy or "pc" is unstable. |
| 605 | * |
| 606 | * This function does "uncharge" from old cgroup but doesn't do "charge" to |
| 607 | * new cgroup. It should be done by a caller. |
| 608 | */ |
| 609 | |
| 610 | static int mem_cgroup_move_account(struct page_cgroup *pc, |
| 611 | struct mem_cgroup *from, struct mem_cgroup *to) |
| 612 | { |
| 613 | struct mem_cgroup_per_zone *from_mz, *to_mz; |
| 614 | int nid, zid; |
| 615 | int ret = -EBUSY; |
| 616 | |
| 617 | VM_BUG_ON(!irqs_disabled()); |
| 618 | VM_BUG_ON(from == to); |
| 619 | |
| 620 | nid = page_cgroup_nid(pc); |
| 621 | zid = page_cgroup_zid(pc); |
| 622 | from_mz = mem_cgroup_zoneinfo(from, nid, zid); |
| 623 | to_mz = mem_cgroup_zoneinfo(to, nid, zid); |
| 624 | |
| 625 | |
| 626 | if (!trylock_page_cgroup(pc)) |
| 627 | return ret; |
| 628 | |
| 629 | if (!PageCgroupUsed(pc)) |
| 630 | goto out; |
| 631 | |
| 632 | if (pc->mem_cgroup != from) |
| 633 | goto out; |
| 634 | |
| 635 | if (spin_trylock(&to_mz->lru_lock)) { |
| 636 | __mem_cgroup_remove_list(from_mz, pc); |
| 637 | css_put(&from->css); |
| 638 | res_counter_uncharge(&from->res, PAGE_SIZE); |
| 639 | pc->mem_cgroup = to; |
| 640 | css_get(&to->css); |
| 641 | __mem_cgroup_add_list(to_mz, pc, false); |
| 642 | ret = 0; |
| 643 | spin_unlock(&to_mz->lru_lock); |
| 644 | } |
| 645 | out: |
| 646 | unlock_page_cgroup(pc); |
| 647 | return ret; |
| 648 | } |
| 649 | |
| 650 | /* |
| 651 | * move charges to its parent. |
| 652 | */ |
| 653 | |
| 654 | static int mem_cgroup_move_parent(struct page_cgroup *pc, |
| 655 | struct mem_cgroup *child, |
| 656 | gfp_t gfp_mask) |
| 657 | { |
| 658 | struct cgroup *cg = child->css.cgroup; |
| 659 | struct cgroup *pcg = cg->parent; |
| 660 | struct mem_cgroup *parent; |
| 661 | struct mem_cgroup_per_zone *mz; |
| 662 | unsigned long flags; |
| 663 | int ret; |
| 664 | |
| 665 | /* Is ROOT ? */ |
| 666 | if (!pcg) |
| 667 | return -EINVAL; |
| 668 | |
| 669 | parent = mem_cgroup_from_cont(pcg); |
| 670 | |
| 671 | ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false); |
| 672 | if (ret) |
| 673 | return ret; |
| 674 | |
| 675 | mz = mem_cgroup_zoneinfo(child, |
| 676 | page_cgroup_nid(pc), page_cgroup_zid(pc)); |
| 677 | |
| 678 | spin_lock_irqsave(&mz->lru_lock, flags); |
| 679 | ret = mem_cgroup_move_account(pc, child, parent); |
| 680 | spin_unlock_irqrestore(&mz->lru_lock, flags); |
| 681 | |
| 682 | /* drop extra refcnt */ |
| 683 | css_put(&parent->css); |
| 684 | /* uncharge if move fails */ |
| 685 | if (ret) |
| 686 | res_counter_uncharge(&parent->res, PAGE_SIZE); |
| 687 | |
| 688 | return ret; |
| 689 | } |
| 690 | |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 691 | /* |
| 692 | * Charge the memory controller for page usage. |
| 693 | * Return |
| 694 | * 0 if the charge was successful |
| 695 | * < 0 if the cgroup is over its limit |
| 696 | */ |
| 697 | static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm, |
| 698 | gfp_t gfp_mask, enum charge_type ctype, |
| 699 | struct mem_cgroup *memcg) |
| 700 | { |
| 701 | struct mem_cgroup *mem; |
| 702 | struct page_cgroup *pc; |
| 703 | int ret; |
| 704 | |
| 705 | pc = lookup_page_cgroup(page); |
| 706 | /* can happen at boot */ |
| 707 | if (unlikely(!pc)) |
| 708 | return 0; |
| 709 | prefetchw(pc); |
| 710 | |
| 711 | mem = memcg; |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 712 | ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true); |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 713 | if (ret) |
| 714 | return ret; |
| 715 | |
| 716 | __mem_cgroup_commit_charge(mem, pc, ctype); |
| 717 | return 0; |
| 718 | } |
| 719 | |
| 720 | int mem_cgroup_newpage_charge(struct page *page, |
| 721 | struct mm_struct *mm, gfp_t gfp_mask) |
KAMEZAWA Hiroyuki | 217bc31 | 2008-02-07 00:14:17 -0800 | [diff] [blame] | 722 | { |
Li Zefan | cede86a | 2008-07-25 01:47:18 -0700 | [diff] [blame] | 723 | if (mem_cgroup_subsys.disabled) |
| 724 | return 0; |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 725 | if (PageCompound(page)) |
| 726 | return 0; |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 727 | /* |
| 728 | * If already mapped, we don't have to account. |
| 729 | * If page cache, page->mapping has address_space. |
| 730 | * But page->mapping may have out-of-use anon_vma pointer, |
| 731 | * detecit it by PageAnon() check. newly-mapped-anon's page->mapping |
| 732 | * is NULL. |
| 733 | */ |
| 734 | if (page_mapped(page) || (page->mapping && !PageAnon(page))) |
| 735 | return 0; |
| 736 | if (unlikely(!mm)) |
| 737 | mm = &init_mm; |
KAMEZAWA Hiroyuki | 217bc31 | 2008-02-07 00:14:17 -0800 | [diff] [blame] | 738 | return mem_cgroup_charge_common(page, mm, gfp_mask, |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 739 | MEM_CGROUP_CHARGE_TYPE_MAPPED, NULL); |
KAMEZAWA Hiroyuki | 217bc31 | 2008-02-07 00:14:17 -0800 | [diff] [blame] | 740 | } |
| 741 | |
Balbir Singh | e1a1cd5 | 2008-02-07 00:14:02 -0800 | [diff] [blame] | 742 | int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm, |
| 743 | gfp_t gfp_mask) |
Balbir Singh | 8697d33 | 2008-02-07 00:13:59 -0800 | [diff] [blame] | 744 | { |
Li Zefan | cede86a | 2008-07-25 01:47:18 -0700 | [diff] [blame] | 745 | if (mem_cgroup_subsys.disabled) |
| 746 | return 0; |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 747 | if (PageCompound(page)) |
| 748 | return 0; |
KAMEZAWA Hiroyuki | accf163 | 2008-07-25 01:47:17 -0700 | [diff] [blame] | 749 | /* |
| 750 | * Corner case handling. This is called from add_to_page_cache() |
| 751 | * in usual. But some FS (shmem) precharges this page before calling it |
| 752 | * and call add_to_page_cache() with GFP_NOWAIT. |
| 753 | * |
| 754 | * For GFP_NOWAIT case, the page may be pre-charged before calling |
| 755 | * add_to_page_cache(). (See shmem.c) check it here and avoid to call |
| 756 | * charge twice. (It works but has to pay a bit larger cost.) |
| 757 | */ |
| 758 | if (!(gfp_mask & __GFP_WAIT)) { |
| 759 | struct page_cgroup *pc; |
| 760 | |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 761 | |
| 762 | pc = lookup_page_cgroup(page); |
| 763 | if (!pc) |
| 764 | return 0; |
| 765 | lock_page_cgroup(pc); |
| 766 | if (PageCgroupUsed(pc)) { |
| 767 | unlock_page_cgroup(pc); |
KAMEZAWA Hiroyuki | accf163 | 2008-07-25 01:47:17 -0700 | [diff] [blame] | 768 | return 0; |
| 769 | } |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 770 | unlock_page_cgroup(pc); |
KAMEZAWA Hiroyuki | accf163 | 2008-07-25 01:47:17 -0700 | [diff] [blame] | 771 | } |
| 772 | |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 773 | if (unlikely(!mm)) |
Balbir Singh | 8697d33 | 2008-02-07 00:13:59 -0800 | [diff] [blame] | 774 | mm = &init_mm; |
KAMEZAWA Hiroyuki | accf163 | 2008-07-25 01:47:17 -0700 | [diff] [blame] | 775 | |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 776 | if (page_is_file_cache(page)) |
| 777 | return mem_cgroup_charge_common(page, mm, gfp_mask, |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 778 | MEM_CGROUP_CHARGE_TYPE_CACHE, NULL); |
KAMEZAWA Hiroyuki | c05555b | 2008-10-18 20:28:11 -0700 | [diff] [blame] | 779 | else |
| 780 | return mem_cgroup_charge_common(page, mm, gfp_mask, |
| 781 | MEM_CGROUP_CHARGE_TYPE_SHMEM, NULL); |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 782 | } |
| 783 | |
KAMEZAWA Hiroyuki | 7a81b88 | 2009-01-07 18:07:48 -0800 | [diff] [blame] | 784 | void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr) |
| 785 | { |
| 786 | struct page_cgroup *pc; |
| 787 | |
| 788 | if (mem_cgroup_subsys.disabled) |
| 789 | return; |
| 790 | if (!ptr) |
| 791 | return; |
| 792 | pc = lookup_page_cgroup(page); |
| 793 | __mem_cgroup_commit_charge(ptr, pc, MEM_CGROUP_CHARGE_TYPE_MAPPED); |
| 794 | } |
| 795 | |
| 796 | void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem) |
| 797 | { |
| 798 | if (mem_cgroup_subsys.disabled) |
| 799 | return; |
| 800 | if (!mem) |
| 801 | return; |
| 802 | res_counter_uncharge(&mem->res, PAGE_SIZE); |
| 803 | css_put(&mem->css); |
| 804 | } |
| 805 | |
| 806 | |
Balbir Singh | 8697d33 | 2008-02-07 00:13:59 -0800 | [diff] [blame] | 807 | /* |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 808 | * uncharge if !page_mapped(page) |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 809 | */ |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 810 | static void |
| 811 | __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype) |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 812 | { |
Hugh Dickins | 8289546 | 2008-03-04 14:29:08 -0800 | [diff] [blame] | 813 | struct page_cgroup *pc; |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 814 | struct mem_cgroup *mem; |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 815 | struct mem_cgroup_per_zone *mz; |
Balbir Singh | 66e1707 | 2008-02-07 00:13:56 -0800 | [diff] [blame] | 816 | unsigned long flags; |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 817 | |
Balbir Singh | 4077960 | 2008-04-04 14:29:59 -0700 | [diff] [blame] | 818 | if (mem_cgroup_subsys.disabled) |
| 819 | return; |
| 820 | |
Balbir Singh | 8697d33 | 2008-02-07 00:13:59 -0800 | [diff] [blame] | 821 | /* |
Balbir Singh | 3c541e1 | 2008-02-07 00:14:41 -0800 | [diff] [blame] | 822 | * Check if our page_cgroup is valid |
Balbir Singh | 8697d33 | 2008-02-07 00:13:59 -0800 | [diff] [blame] | 823 | */ |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 824 | pc = lookup_page_cgroup(page); |
| 825 | if (unlikely(!pc || !PageCgroupUsed(pc))) |
| 826 | return; |
Balbir Singh | 8a9f3cc | 2008-02-07 00:13:53 -0800 | [diff] [blame] | 827 | |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 828 | lock_page_cgroup(pc); |
| 829 | if ((ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED && page_mapped(page)) |
| 830 | || !PageCgroupUsed(pc)) { |
| 831 | /* This happens at race in zap_pte_range() and do_swap_page()*/ |
| 832 | unlock_page_cgroup(pc); |
| 833 | return; |
| 834 | } |
| 835 | ClearPageCgroupUsed(pc); |
| 836 | mem = pc->mem_cgroup; |
Hugh Dickins | b9c565d | 2008-03-04 14:29:11 -0800 | [diff] [blame] | 837 | |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 838 | mz = page_cgroup_zoneinfo(pc); |
| 839 | spin_lock_irqsave(&mz->lru_lock, flags); |
| 840 | __mem_cgroup_remove_list(mz, pc); |
| 841 | spin_unlock_irqrestore(&mz->lru_lock, flags); |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 842 | unlock_page_cgroup(pc); |
Hugh Dickins | fb59e9f | 2008-03-04 14:29:16 -0800 | [diff] [blame] | 843 | |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 844 | res_counter_uncharge(&mem->res, PAGE_SIZE); |
| 845 | css_put(&mem->css); |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 846 | |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 847 | return; |
Balbir Singh | 3c541e1 | 2008-02-07 00:14:41 -0800 | [diff] [blame] | 848 | } |
| 849 | |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 850 | void mem_cgroup_uncharge_page(struct page *page) |
| 851 | { |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 852 | /* early check. */ |
| 853 | if (page_mapped(page)) |
| 854 | return; |
| 855 | if (page->mapping && !PageAnon(page)) |
| 856 | return; |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 857 | __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED); |
| 858 | } |
| 859 | |
| 860 | void mem_cgroup_uncharge_cache_page(struct page *page) |
| 861 | { |
| 862 | VM_BUG_ON(page_mapped(page)); |
KAMEZAWA Hiroyuki | b7abea9 | 2008-10-18 20:28:09 -0700 | [diff] [blame] | 863 | VM_BUG_ON(page->mapping); |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 864 | __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE); |
| 865 | } |
| 866 | |
KAMEZAWA Hiroyuki | ae41be3 | 2008-02-07 00:14:10 -0800 | [diff] [blame] | 867 | /* |
KAMEZAWA Hiroyuki | 01b1ae6 | 2009-01-07 18:07:50 -0800 | [diff] [blame] | 868 | * Before starting migration, account PAGE_SIZE to mem_cgroup that the old |
| 869 | * page belongs to. |
KAMEZAWA Hiroyuki | ae41be3 | 2008-02-07 00:14:10 -0800 | [diff] [blame] | 870 | */ |
KAMEZAWA Hiroyuki | 01b1ae6 | 2009-01-07 18:07:50 -0800 | [diff] [blame] | 871 | int mem_cgroup_prepare_migration(struct page *page, struct mem_cgroup **ptr) |
KAMEZAWA Hiroyuki | ae41be3 | 2008-02-07 00:14:10 -0800 | [diff] [blame] | 872 | { |
| 873 | struct page_cgroup *pc; |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 874 | struct mem_cgroup *mem = NULL; |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 875 | int ret = 0; |
Hugh Dickins | 8869b8f | 2008-03-04 14:29:09 -0800 | [diff] [blame] | 876 | |
Balbir Singh | 4077960 | 2008-04-04 14:29:59 -0700 | [diff] [blame] | 877 | if (mem_cgroup_subsys.disabled) |
| 878 | return 0; |
| 879 | |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 880 | pc = lookup_page_cgroup(page); |
| 881 | lock_page_cgroup(pc); |
| 882 | if (PageCgroupUsed(pc)) { |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 883 | mem = pc->mem_cgroup; |
| 884 | css_get(&mem->css); |
Hugh Dickins | b9c565d | 2008-03-04 14:29:11 -0800 | [diff] [blame] | 885 | } |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 886 | unlock_page_cgroup(pc); |
KAMEZAWA Hiroyuki | 01b1ae6 | 2009-01-07 18:07:50 -0800 | [diff] [blame] | 887 | |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 888 | if (mem) { |
KAMEZAWA Hiroyuki | 01b1ae6 | 2009-01-07 18:07:50 -0800 | [diff] [blame] | 889 | ret = mem_cgroup_try_charge(NULL, GFP_HIGHUSER_MOVABLE, &mem); |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 890 | css_put(&mem->css); |
| 891 | } |
KAMEZAWA Hiroyuki | 01b1ae6 | 2009-01-07 18:07:50 -0800 | [diff] [blame] | 892 | *ptr = mem; |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 893 | return ret; |
| 894 | } |
Hugh Dickins | fb59e9f | 2008-03-04 14:29:16 -0800 | [diff] [blame] | 895 | |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 896 | /* remove redundant charge if migration failed*/ |
KAMEZAWA Hiroyuki | 01b1ae6 | 2009-01-07 18:07:50 -0800 | [diff] [blame] | 897 | void mem_cgroup_end_migration(struct mem_cgroup *mem, |
| 898 | struct page *oldpage, struct page *newpage) |
KAMEZAWA Hiroyuki | e8589cc | 2008-07-25 01:47:10 -0700 | [diff] [blame] | 899 | { |
KAMEZAWA Hiroyuki | 01b1ae6 | 2009-01-07 18:07:50 -0800 | [diff] [blame] | 900 | struct page *target, *unused; |
| 901 | struct page_cgroup *pc; |
| 902 | enum charge_type ctype; |
| 903 | |
| 904 | if (!mem) |
| 905 | return; |
| 906 | |
| 907 | /* at migration success, oldpage->mapping is NULL. */ |
| 908 | if (oldpage->mapping) { |
| 909 | target = oldpage; |
| 910 | unused = NULL; |
| 911 | } else { |
| 912 | target = newpage; |
| 913 | unused = oldpage; |
| 914 | } |
| 915 | |
| 916 | if (PageAnon(target)) |
| 917 | ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED; |
| 918 | else if (page_is_file_cache(target)) |
| 919 | ctype = MEM_CGROUP_CHARGE_TYPE_CACHE; |
| 920 | else |
| 921 | ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM; |
| 922 | |
| 923 | /* unused page is not on radix-tree now. */ |
| 924 | if (unused && ctype != MEM_CGROUP_CHARGE_TYPE_MAPPED) |
| 925 | __mem_cgroup_uncharge_common(unused, ctype); |
| 926 | |
| 927 | pc = lookup_page_cgroup(target); |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 928 | /* |
KAMEZAWA Hiroyuki | 01b1ae6 | 2009-01-07 18:07:50 -0800 | [diff] [blame] | 929 | * __mem_cgroup_commit_charge() check PCG_USED bit of page_cgroup. |
| 930 | * So, double-counting is effectively avoided. |
KAMEZAWA Hiroyuki | 69029cd | 2008-07-25 01:47:14 -0700 | [diff] [blame] | 931 | */ |
KAMEZAWA Hiroyuki | 01b1ae6 | 2009-01-07 18:07:50 -0800 | [diff] [blame] | 932 | __mem_cgroup_commit_charge(mem, pc, ctype); |
| 933 | |
| 934 | /* |
| 935 | * Both of oldpage and newpage are still under lock_page(). |
| 936 | * Then, we don't have to care about race in radix-tree. |
| 937 | * But we have to be careful that this page is unmapped or not. |
| 938 | * |
| 939 | * There is a case for !page_mapped(). At the start of |
| 940 | * migration, oldpage was mapped. But now, it's zapped. |
| 941 | * But we know *target* page is not freed/reused under us. |
| 942 | * mem_cgroup_uncharge_page() does all necessary checks. |
| 943 | */ |
| 944 | if (ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED) |
| 945 | mem_cgroup_uncharge_page(target); |
KAMEZAWA Hiroyuki | ae41be3 | 2008-02-07 00:14:10 -0800 | [diff] [blame] | 946 | } |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 947 | |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 948 | /* |
KAMEZAWA Hiroyuki | c9b0ed5 | 2008-07-25 01:47:15 -0700 | [diff] [blame] | 949 | * A call to try to shrink memory usage under specified resource controller. |
| 950 | * This is typically used for page reclaiming for shmem for reducing side |
| 951 | * effect of page allocation from shmem, which is used by some mem_cgroup. |
| 952 | */ |
| 953 | int mem_cgroup_shrink_usage(struct mm_struct *mm, gfp_t gfp_mask) |
| 954 | { |
| 955 | struct mem_cgroup *mem; |
| 956 | int progress = 0; |
| 957 | int retry = MEM_CGROUP_RECLAIM_RETRIES; |
| 958 | |
Li Zefan | cede86a | 2008-07-25 01:47:18 -0700 | [diff] [blame] | 959 | if (mem_cgroup_subsys.disabled) |
| 960 | return 0; |
Hugh Dickins | 9623e07 | 2008-08-12 15:08:41 -0700 | [diff] [blame] | 961 | if (!mm) |
| 962 | return 0; |
Li Zefan | cede86a | 2008-07-25 01:47:18 -0700 | [diff] [blame] | 963 | |
KAMEZAWA Hiroyuki | c9b0ed5 | 2008-07-25 01:47:15 -0700 | [diff] [blame] | 964 | rcu_read_lock(); |
| 965 | mem = mem_cgroup_from_task(rcu_dereference(mm->owner)); |
Balbir Singh | 31a78f2 | 2008-09-28 23:09:31 +0100 | [diff] [blame] | 966 | if (unlikely(!mem)) { |
| 967 | rcu_read_unlock(); |
| 968 | return 0; |
| 969 | } |
KAMEZAWA Hiroyuki | c9b0ed5 | 2008-07-25 01:47:15 -0700 | [diff] [blame] | 970 | css_get(&mem->css); |
| 971 | rcu_read_unlock(); |
| 972 | |
| 973 | do { |
| 974 | progress = try_to_free_mem_cgroup_pages(mem, gfp_mask); |
Daisuke Nishimura | a10cebf | 2008-09-22 13:57:52 -0700 | [diff] [blame] | 975 | progress += res_counter_check_under_limit(&mem->res); |
KAMEZAWA Hiroyuki | c9b0ed5 | 2008-07-25 01:47:15 -0700 | [diff] [blame] | 976 | } while (!progress && --retry); |
| 977 | |
| 978 | css_put(&mem->css); |
| 979 | if (!retry) |
| 980 | return -ENOMEM; |
| 981 | return 0; |
| 982 | } |
| 983 | |
KOSAKI Motohiro | d38d2a7 | 2009-01-06 14:39:44 -0800 | [diff] [blame] | 984 | static int mem_cgroup_resize_limit(struct mem_cgroup *memcg, |
| 985 | unsigned long long val) |
KAMEZAWA Hiroyuki | 628f423 | 2008-07-25 01:47:20 -0700 | [diff] [blame] | 986 | { |
| 987 | |
| 988 | int retry_count = MEM_CGROUP_RECLAIM_RETRIES; |
| 989 | int progress; |
| 990 | int ret = 0; |
| 991 | |
| 992 | while (res_counter_set_limit(&memcg->res, val)) { |
| 993 | if (signal_pending(current)) { |
| 994 | ret = -EINTR; |
| 995 | break; |
| 996 | } |
| 997 | if (!retry_count) { |
| 998 | ret = -EBUSY; |
| 999 | break; |
| 1000 | } |
KAMEZAWA Hiroyuki | bced052 | 2009-01-07 18:07:49 -0800 | [diff] [blame] | 1001 | progress = try_to_free_mem_cgroup_pages(memcg, |
| 1002 | GFP_HIGHUSER_MOVABLE); |
KAMEZAWA Hiroyuki | 628f423 | 2008-07-25 01:47:20 -0700 | [diff] [blame] | 1003 | if (!progress) |
| 1004 | retry_count--; |
| 1005 | } |
| 1006 | return ret; |
| 1007 | } |
| 1008 | |
| 1009 | |
KAMEZAWA Hiroyuki | c9b0ed5 | 2008-07-25 01:47:15 -0700 | [diff] [blame] | 1010 | /* |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 1011 | * This routine traverse page_cgroup in given list and drop them all. |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 1012 | * *And* this routine doesn't reclaim page itself, just removes page_cgroup. |
| 1013 | */ |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1014 | static int mem_cgroup_force_empty_list(struct mem_cgroup *mem, |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 1015 | struct mem_cgroup_per_zone *mz, |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 1016 | enum lru_list lru) |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 1017 | { |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1018 | struct page_cgroup *pc, *busy; |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 1019 | unsigned long flags; |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1020 | unsigned long loop; |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 1021 | struct list_head *list; |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1022 | int ret = 0; |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 1023 | |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 1024 | list = &mz->lists[lru]; |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 1025 | |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1026 | loop = MEM_CGROUP_ZSTAT(mz, lru); |
| 1027 | /* give some margin against EBUSY etc...*/ |
| 1028 | loop += 256; |
| 1029 | busy = NULL; |
| 1030 | while (loop--) { |
| 1031 | ret = 0; |
| 1032 | spin_lock_irqsave(&mz->lru_lock, flags); |
| 1033 | if (list_empty(list)) { |
| 1034 | spin_unlock_irqrestore(&mz->lru_lock, flags); |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 1035 | break; |
| 1036 | } |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1037 | pc = list_entry(list->prev, struct page_cgroup, lru); |
| 1038 | if (busy == pc) { |
| 1039 | list_move(&pc->lru, list); |
| 1040 | busy = 0; |
| 1041 | spin_unlock_irqrestore(&mz->lru_lock, flags); |
| 1042 | continue; |
| 1043 | } |
| 1044 | spin_unlock_irqrestore(&mz->lru_lock, flags); |
| 1045 | |
| 1046 | ret = mem_cgroup_move_parent(pc, mem, GFP_HIGHUSER_MOVABLE); |
| 1047 | if (ret == -ENOMEM) |
| 1048 | break; |
| 1049 | |
| 1050 | if (ret == -EBUSY || ret == -EINVAL) { |
| 1051 | /* found lock contention or "pc" is obsolete. */ |
| 1052 | busy = pc; |
| 1053 | cond_resched(); |
| 1054 | } else |
| 1055 | busy = NULL; |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 1056 | } |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1057 | if (!ret && !list_empty(list)) |
| 1058 | return -EBUSY; |
| 1059 | return ret; |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 1060 | } |
| 1061 | |
| 1062 | /* |
| 1063 | * make mem_cgroup's charge to be 0 if there is no task. |
| 1064 | * This enables deleting this mem_cgroup. |
| 1065 | */ |
Hugh Dickins | d5b69e3 | 2008-03-04 14:29:10 -0800 | [diff] [blame] | 1066 | static int mem_cgroup_force_empty(struct mem_cgroup *mem) |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 1067 | { |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1068 | int ret; |
| 1069 | int node, zid, shrink; |
| 1070 | int nr_retries = MEM_CGROUP_RECLAIM_RETRIES; |
Hugh Dickins | 8869b8f | 2008-03-04 14:29:09 -0800 | [diff] [blame] | 1071 | |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 1072 | css_get(&mem->css); |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1073 | |
| 1074 | shrink = 0; |
| 1075 | move_account: |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1076 | while (mem->res.usage > 0) { |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1077 | ret = -EBUSY; |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 1078 | if (atomic_read(&mem->css.cgroup->count) > 0) |
| 1079 | goto out; |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1080 | |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 1081 | /* This is for making all *used* pages to be on LRU. */ |
| 1082 | lru_add_drain_all(); |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1083 | ret = 0; |
| 1084 | for_each_node_state(node, N_POSSIBLE) { |
| 1085 | for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) { |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1086 | struct mem_cgroup_per_zone *mz; |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 1087 | enum lru_list l; |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1088 | mz = mem_cgroup_zoneinfo(mem, node, zid); |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1089 | for_each_lru(l) { |
| 1090 | ret = mem_cgroup_force_empty_list(mem, |
| 1091 | mz, l); |
| 1092 | if (ret) |
| 1093 | break; |
| 1094 | } |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1095 | } |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1096 | if (ret) |
| 1097 | break; |
| 1098 | } |
| 1099 | /* it seems parent cgroup doesn't have enough mem */ |
| 1100 | if (ret == -ENOMEM) |
| 1101 | goto try_to_free; |
KAMEZAWA Hiroyuki | 52d4b9a | 2008-10-18 20:28:16 -0700 | [diff] [blame] | 1102 | cond_resched(); |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 1103 | } |
| 1104 | ret = 0; |
| 1105 | out: |
| 1106 | css_put(&mem->css); |
| 1107 | return ret; |
KAMEZAWA Hiroyuki | f817ed4 | 2009-01-07 18:07:53 -0800 | [diff] [blame^] | 1108 | |
| 1109 | try_to_free: |
| 1110 | /* returns EBUSY if we come here twice. */ |
| 1111 | if (shrink) { |
| 1112 | ret = -EBUSY; |
| 1113 | goto out; |
| 1114 | } |
| 1115 | /* try to free all pages in this cgroup */ |
| 1116 | shrink = 1; |
| 1117 | while (nr_retries && mem->res.usage > 0) { |
| 1118 | int progress; |
| 1119 | progress = try_to_free_mem_cgroup_pages(mem, |
| 1120 | GFP_HIGHUSER_MOVABLE); |
| 1121 | if (!progress) |
| 1122 | nr_retries--; |
| 1123 | |
| 1124 | } |
| 1125 | /* try move_account...there may be some *locked* pages. */ |
| 1126 | if (mem->res.usage) |
| 1127 | goto move_account; |
| 1128 | ret = 0; |
| 1129 | goto out; |
KAMEZAWA Hiroyuki | cc84758 | 2008-02-07 00:14:16 -0800 | [diff] [blame] | 1130 | } |
| 1131 | |
Paul Menage | 2c3daa7 | 2008-04-29 00:59:58 -0700 | [diff] [blame] | 1132 | static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft) |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1133 | { |
Paul Menage | 2c3daa7 | 2008-04-29 00:59:58 -0700 | [diff] [blame] | 1134 | return res_counter_read_u64(&mem_cgroup_from_cont(cont)->res, |
| 1135 | cft->private); |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1136 | } |
KAMEZAWA Hiroyuki | 628f423 | 2008-07-25 01:47:20 -0700 | [diff] [blame] | 1137 | /* |
| 1138 | * The user of this function is... |
| 1139 | * RES_LIMIT. |
| 1140 | */ |
Paul Menage | 856c13a | 2008-07-25 01:47:04 -0700 | [diff] [blame] | 1141 | static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft, |
| 1142 | const char *buffer) |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1143 | { |
KAMEZAWA Hiroyuki | 628f423 | 2008-07-25 01:47:20 -0700 | [diff] [blame] | 1144 | struct mem_cgroup *memcg = mem_cgroup_from_cont(cont); |
| 1145 | unsigned long long val; |
| 1146 | int ret; |
| 1147 | |
| 1148 | switch (cft->private) { |
| 1149 | case RES_LIMIT: |
| 1150 | /* This function does all necessary parse...reuse it */ |
| 1151 | ret = res_counter_memparse_write_strategy(buffer, &val); |
| 1152 | if (!ret) |
| 1153 | ret = mem_cgroup_resize_limit(memcg, val); |
| 1154 | break; |
| 1155 | default: |
| 1156 | ret = -EINVAL; /* should be BUG() ? */ |
| 1157 | break; |
| 1158 | } |
| 1159 | return ret; |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1160 | } |
| 1161 | |
Pavel Emelyanov | 29f2a4d | 2008-04-29 01:00:21 -0700 | [diff] [blame] | 1162 | static int mem_cgroup_reset(struct cgroup *cont, unsigned int event) |
Pavel Emelyanov | c84872e | 2008-04-29 01:00:17 -0700 | [diff] [blame] | 1163 | { |
| 1164 | struct mem_cgroup *mem; |
| 1165 | |
| 1166 | mem = mem_cgroup_from_cont(cont); |
Pavel Emelyanov | 29f2a4d | 2008-04-29 01:00:21 -0700 | [diff] [blame] | 1167 | switch (event) { |
| 1168 | case RES_MAX_USAGE: |
| 1169 | res_counter_reset_max(&mem->res); |
| 1170 | break; |
| 1171 | case RES_FAILCNT: |
| 1172 | res_counter_reset_failcnt(&mem->res); |
| 1173 | break; |
| 1174 | } |
Pavel Emelyanov | 85cc59d | 2008-04-29 01:00:20 -0700 | [diff] [blame] | 1175 | return 0; |
Pavel Emelyanov | c84872e | 2008-04-29 01:00:17 -0700 | [diff] [blame] | 1176 | } |
| 1177 | |
KAMEZAWA Hiroyuki | d2ceb9b | 2008-02-07 00:14:25 -0800 | [diff] [blame] | 1178 | static const struct mem_cgroup_stat_desc { |
| 1179 | const char *msg; |
| 1180 | u64 unit; |
| 1181 | } mem_cgroup_stat_desc[] = { |
| 1182 | [MEM_CGROUP_STAT_CACHE] = { "cache", PAGE_SIZE, }, |
| 1183 | [MEM_CGROUP_STAT_RSS] = { "rss", PAGE_SIZE, }, |
Balaji Rao | 55e462b | 2008-05-01 04:35:12 -0700 | [diff] [blame] | 1184 | [MEM_CGROUP_STAT_PGPGIN_COUNT] = {"pgpgin", 1, }, |
| 1185 | [MEM_CGROUP_STAT_PGPGOUT_COUNT] = {"pgpgout", 1, }, |
KAMEZAWA Hiroyuki | d2ceb9b | 2008-02-07 00:14:25 -0800 | [diff] [blame] | 1186 | }; |
| 1187 | |
Paul Menage | c64745c | 2008-04-29 01:00:02 -0700 | [diff] [blame] | 1188 | static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft, |
| 1189 | struct cgroup_map_cb *cb) |
KAMEZAWA Hiroyuki | d2ceb9b | 2008-02-07 00:14:25 -0800 | [diff] [blame] | 1190 | { |
KAMEZAWA Hiroyuki | d2ceb9b | 2008-02-07 00:14:25 -0800 | [diff] [blame] | 1191 | struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont); |
| 1192 | struct mem_cgroup_stat *stat = &mem_cont->stat; |
| 1193 | int i; |
| 1194 | |
| 1195 | for (i = 0; i < ARRAY_SIZE(stat->cpustat[0].count); i++) { |
| 1196 | s64 val; |
| 1197 | |
| 1198 | val = mem_cgroup_read_stat(stat, i); |
| 1199 | val *= mem_cgroup_stat_desc[i].unit; |
Paul Menage | c64745c | 2008-04-29 01:00:02 -0700 | [diff] [blame] | 1200 | cb->fill(cb, mem_cgroup_stat_desc[i].msg, val); |
KAMEZAWA Hiroyuki | d2ceb9b | 2008-02-07 00:14:25 -0800 | [diff] [blame] | 1201 | } |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1202 | /* showing # of active pages */ |
| 1203 | { |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 1204 | unsigned long active_anon, inactive_anon; |
| 1205 | unsigned long active_file, inactive_file; |
Lee Schermerhorn | 7b85412 | 2008-10-18 20:26:40 -0700 | [diff] [blame] | 1206 | unsigned long unevictable; |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1207 | |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 1208 | inactive_anon = mem_cgroup_get_all_zonestat(mem_cont, |
| 1209 | LRU_INACTIVE_ANON); |
| 1210 | active_anon = mem_cgroup_get_all_zonestat(mem_cont, |
| 1211 | LRU_ACTIVE_ANON); |
| 1212 | inactive_file = mem_cgroup_get_all_zonestat(mem_cont, |
| 1213 | LRU_INACTIVE_FILE); |
| 1214 | active_file = mem_cgroup_get_all_zonestat(mem_cont, |
| 1215 | LRU_ACTIVE_FILE); |
Lee Schermerhorn | 7b85412 | 2008-10-18 20:26:40 -0700 | [diff] [blame] | 1216 | unevictable = mem_cgroup_get_all_zonestat(mem_cont, |
| 1217 | LRU_UNEVICTABLE); |
| 1218 | |
Rik van Riel | 4f98a2f | 2008-10-18 20:26:32 -0700 | [diff] [blame] | 1219 | cb->fill(cb, "active_anon", (active_anon) * PAGE_SIZE); |
| 1220 | cb->fill(cb, "inactive_anon", (inactive_anon) * PAGE_SIZE); |
| 1221 | cb->fill(cb, "active_file", (active_file) * PAGE_SIZE); |
| 1222 | cb->fill(cb, "inactive_file", (inactive_file) * PAGE_SIZE); |
Lee Schermerhorn | 7b85412 | 2008-10-18 20:26:40 -0700 | [diff] [blame] | 1223 | cb->fill(cb, "unevictable", unevictable * PAGE_SIZE); |
| 1224 | |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1225 | } |
KAMEZAWA Hiroyuki | d2ceb9b | 2008-02-07 00:14:25 -0800 | [diff] [blame] | 1226 | return 0; |
| 1227 | } |
| 1228 | |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1229 | static struct cftype mem_cgroup_files[] = { |
| 1230 | { |
Balbir Singh | 0eea103 | 2008-02-07 00:13:57 -0800 | [diff] [blame] | 1231 | .name = "usage_in_bytes", |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1232 | .private = RES_USAGE, |
Paul Menage | 2c3daa7 | 2008-04-29 00:59:58 -0700 | [diff] [blame] | 1233 | .read_u64 = mem_cgroup_read, |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1234 | }, |
| 1235 | { |
Pavel Emelyanov | c84872e | 2008-04-29 01:00:17 -0700 | [diff] [blame] | 1236 | .name = "max_usage_in_bytes", |
| 1237 | .private = RES_MAX_USAGE, |
Pavel Emelyanov | 29f2a4d | 2008-04-29 01:00:21 -0700 | [diff] [blame] | 1238 | .trigger = mem_cgroup_reset, |
Pavel Emelyanov | c84872e | 2008-04-29 01:00:17 -0700 | [diff] [blame] | 1239 | .read_u64 = mem_cgroup_read, |
| 1240 | }, |
| 1241 | { |
Balbir Singh | 0eea103 | 2008-02-07 00:13:57 -0800 | [diff] [blame] | 1242 | .name = "limit_in_bytes", |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1243 | .private = RES_LIMIT, |
Paul Menage | 856c13a | 2008-07-25 01:47:04 -0700 | [diff] [blame] | 1244 | .write_string = mem_cgroup_write, |
Paul Menage | 2c3daa7 | 2008-04-29 00:59:58 -0700 | [diff] [blame] | 1245 | .read_u64 = mem_cgroup_read, |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1246 | }, |
| 1247 | { |
| 1248 | .name = "failcnt", |
| 1249 | .private = RES_FAILCNT, |
Pavel Emelyanov | 29f2a4d | 2008-04-29 01:00:21 -0700 | [diff] [blame] | 1250 | .trigger = mem_cgroup_reset, |
Paul Menage | 2c3daa7 | 2008-04-29 00:59:58 -0700 | [diff] [blame] | 1251 | .read_u64 = mem_cgroup_read, |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1252 | }, |
Balbir Singh | 8697d33 | 2008-02-07 00:13:59 -0800 | [diff] [blame] | 1253 | { |
KAMEZAWA Hiroyuki | d2ceb9b | 2008-02-07 00:14:25 -0800 | [diff] [blame] | 1254 | .name = "stat", |
Paul Menage | c64745c | 2008-04-29 01:00:02 -0700 | [diff] [blame] | 1255 | .read_map = mem_control_stat_show, |
KAMEZAWA Hiroyuki | d2ceb9b | 2008-02-07 00:14:25 -0800 | [diff] [blame] | 1256 | }, |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1257 | }; |
| 1258 | |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1259 | static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node) |
| 1260 | { |
| 1261 | struct mem_cgroup_per_node *pn; |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1262 | struct mem_cgroup_per_zone *mz; |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 1263 | enum lru_list l; |
KAMEZAWA Hiroyuki | 41e3355 | 2008-04-08 17:41:54 -0700 | [diff] [blame] | 1264 | int zone, tmp = node; |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1265 | /* |
| 1266 | * This routine is called against possible nodes. |
| 1267 | * But it's BUG to call kmalloc() against offline node. |
| 1268 | * |
| 1269 | * TODO: this routine can waste much memory for nodes which will |
| 1270 | * never be onlined. It's better to use memory hotplug callback |
| 1271 | * function. |
| 1272 | */ |
KAMEZAWA Hiroyuki | 41e3355 | 2008-04-08 17:41:54 -0700 | [diff] [blame] | 1273 | if (!node_state(node, N_NORMAL_MEMORY)) |
| 1274 | tmp = -1; |
| 1275 | pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, tmp); |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1276 | if (!pn) |
| 1277 | return 1; |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1278 | |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1279 | mem->info.nodeinfo[node] = pn; |
| 1280 | memset(pn, 0, sizeof(*pn)); |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1281 | |
| 1282 | for (zone = 0; zone < MAX_NR_ZONES; zone++) { |
| 1283 | mz = &pn->zoneinfo[zone]; |
KAMEZAWA Hiroyuki | 072c56c1 | 2008-02-07 00:14:39 -0800 | [diff] [blame] | 1284 | spin_lock_init(&mz->lru_lock); |
Christoph Lameter | b69408e | 2008-10-18 20:26:14 -0700 | [diff] [blame] | 1285 | for_each_lru(l) |
| 1286 | INIT_LIST_HEAD(&mz->lists[l]); |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1287 | } |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1288 | return 0; |
| 1289 | } |
| 1290 | |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1291 | static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node) |
| 1292 | { |
| 1293 | kfree(mem->info.nodeinfo[node]); |
| 1294 | } |
| 1295 | |
KAMEZAWA Hiroyuki | 3332794 | 2008-04-29 01:00:24 -0700 | [diff] [blame] | 1296 | static struct mem_cgroup *mem_cgroup_alloc(void) |
| 1297 | { |
| 1298 | struct mem_cgroup *mem; |
| 1299 | |
| 1300 | if (sizeof(*mem) < PAGE_SIZE) |
| 1301 | mem = kmalloc(sizeof(*mem), GFP_KERNEL); |
| 1302 | else |
| 1303 | mem = vmalloc(sizeof(*mem)); |
| 1304 | |
| 1305 | if (mem) |
| 1306 | memset(mem, 0, sizeof(*mem)); |
| 1307 | return mem; |
| 1308 | } |
| 1309 | |
| 1310 | static void mem_cgroup_free(struct mem_cgroup *mem) |
| 1311 | { |
| 1312 | if (sizeof(*mem) < PAGE_SIZE) |
| 1313 | kfree(mem); |
| 1314 | else |
| 1315 | vfree(mem); |
| 1316 | } |
| 1317 | |
| 1318 | |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1319 | static struct cgroup_subsys_state * |
| 1320 | mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont) |
| 1321 | { |
| 1322 | struct mem_cgroup *mem; |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1323 | int node; |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1324 | |
Balbir Singh | b6ac57d | 2008-04-29 01:00:19 -0700 | [diff] [blame] | 1325 | if (unlikely((cont->parent) == NULL)) { |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 1326 | mem = &init_mem_cgroup; |
Balbir Singh | b6ac57d | 2008-04-29 01:00:19 -0700 | [diff] [blame] | 1327 | } else { |
KAMEZAWA Hiroyuki | 3332794 | 2008-04-29 01:00:24 -0700 | [diff] [blame] | 1328 | mem = mem_cgroup_alloc(); |
| 1329 | if (!mem) |
| 1330 | return ERR_PTR(-ENOMEM); |
Balbir Singh | b6ac57d | 2008-04-29 01:00:19 -0700 | [diff] [blame] | 1331 | } |
Pavel Emelianov | 78fb746 | 2008-02-07 00:13:51 -0800 | [diff] [blame] | 1332 | |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1333 | res_counter_init(&mem->res); |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1334 | |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1335 | for_each_node_state(node, N_POSSIBLE) |
| 1336 | if (alloc_mem_cgroup_per_zone_info(mem, node)) |
| 1337 | goto free_out; |
| 1338 | |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1339 | return &mem->css; |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1340 | free_out: |
| 1341 | for_each_node_state(node, N_POSSIBLE) |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1342 | free_mem_cgroup_per_zone_info(mem, node); |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1343 | if (cont->parent != NULL) |
KAMEZAWA Hiroyuki | 3332794 | 2008-04-29 01:00:24 -0700 | [diff] [blame] | 1344 | mem_cgroup_free(mem); |
Li Zefan | 2dda81c | 2008-02-23 15:24:14 -0800 | [diff] [blame] | 1345 | return ERR_PTR(-ENOMEM); |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1346 | } |
| 1347 | |
KAMEZAWA Hiroyuki | df878fb | 2008-02-07 00:14:28 -0800 | [diff] [blame] | 1348 | static void mem_cgroup_pre_destroy(struct cgroup_subsys *ss, |
| 1349 | struct cgroup *cont) |
| 1350 | { |
| 1351 | struct mem_cgroup *mem = mem_cgroup_from_cont(cont); |
| 1352 | mem_cgroup_force_empty(mem); |
| 1353 | } |
| 1354 | |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1355 | static void mem_cgroup_destroy(struct cgroup_subsys *ss, |
| 1356 | struct cgroup *cont) |
| 1357 | { |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1358 | int node; |
| 1359 | struct mem_cgroup *mem = mem_cgroup_from_cont(cont); |
| 1360 | |
| 1361 | for_each_node_state(node, N_POSSIBLE) |
KAMEZAWA Hiroyuki | 1ecaab2 | 2008-02-07 00:14:38 -0800 | [diff] [blame] | 1362 | free_mem_cgroup_per_zone_info(mem, node); |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1363 | |
KAMEZAWA Hiroyuki | 3332794 | 2008-04-29 01:00:24 -0700 | [diff] [blame] | 1364 | mem_cgroup_free(mem_cgroup_from_cont(cont)); |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1365 | } |
| 1366 | |
| 1367 | static int mem_cgroup_populate(struct cgroup_subsys *ss, |
| 1368 | struct cgroup *cont) |
| 1369 | { |
| 1370 | return cgroup_add_files(cont, ss, mem_cgroup_files, |
| 1371 | ARRAY_SIZE(mem_cgroup_files)); |
| 1372 | } |
| 1373 | |
Balbir Singh | 67e465a | 2008-02-07 00:13:54 -0800 | [diff] [blame] | 1374 | static void mem_cgroup_move_task(struct cgroup_subsys *ss, |
| 1375 | struct cgroup *cont, |
| 1376 | struct cgroup *old_cont, |
| 1377 | struct task_struct *p) |
| 1378 | { |
| 1379 | struct mm_struct *mm; |
| 1380 | struct mem_cgroup *mem, *old_mem; |
| 1381 | |
| 1382 | mm = get_task_mm(p); |
| 1383 | if (mm == NULL) |
| 1384 | return; |
| 1385 | |
| 1386 | mem = mem_cgroup_from_cont(cont); |
| 1387 | old_mem = mem_cgroup_from_cont(old_cont); |
| 1388 | |
Balbir Singh | 67e465a | 2008-02-07 00:13:54 -0800 | [diff] [blame] | 1389 | /* |
| 1390 | * Only thread group leaders are allowed to migrate, the mm_struct is |
| 1391 | * in effect owned by the leader |
| 1392 | */ |
Pavel Emelyanov | 52ea27e | 2008-03-19 17:00:45 -0700 | [diff] [blame] | 1393 | if (!thread_group_leader(p)) |
Balbir Singh | 67e465a | 2008-02-07 00:13:54 -0800 | [diff] [blame] | 1394 | goto out; |
| 1395 | |
Balbir Singh | 67e465a | 2008-02-07 00:13:54 -0800 | [diff] [blame] | 1396 | out: |
| 1397 | mmput(mm); |
Balbir Singh | 67e465a | 2008-02-07 00:13:54 -0800 | [diff] [blame] | 1398 | } |
| 1399 | |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1400 | struct cgroup_subsys mem_cgroup_subsys = { |
| 1401 | .name = "memory", |
| 1402 | .subsys_id = mem_cgroup_subsys_id, |
| 1403 | .create = mem_cgroup_create, |
KAMEZAWA Hiroyuki | df878fb | 2008-02-07 00:14:28 -0800 | [diff] [blame] | 1404 | .pre_destroy = mem_cgroup_pre_destroy, |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1405 | .destroy = mem_cgroup_destroy, |
| 1406 | .populate = mem_cgroup_populate, |
Balbir Singh | 67e465a | 2008-02-07 00:13:54 -0800 | [diff] [blame] | 1407 | .attach = mem_cgroup_move_task, |
KAMEZAWA Hiroyuki | 6d12e2d | 2008-02-07 00:14:31 -0800 | [diff] [blame] | 1408 | .early_init = 0, |
Balbir Singh | 8cdea7c | 2008-02-07 00:13:50 -0800 | [diff] [blame] | 1409 | }; |