blob: 866dcc7eeb0c3da1e4c6d822ece39a3ea65dc6f9 [file] [log] [blame]
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001/* memcontrol.c - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08006 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
Balbir Singh8cdea7c2008-02-07 00:13:50 -08009 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 */
19
20#include <linux/res_counter.h>
21#include <linux/memcontrol.h>
22#include <linux/cgroup.h>
Pavel Emelianov78fb7462008-02-07 00:13:51 -080023#include <linux/mm.h>
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080024#include <linux/smp.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080025#include <linux/page-flags.h>
Balbir Singh66e17072008-02-07 00:13:56 -080026#include <linux/backing-dev.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080027#include <linux/bit_spinlock.h>
28#include <linux/rcupdate.h>
Balbir Singhb6ac57d2008-04-29 01:00:19 -070029#include <linux/slab.h>
Balbir Singh66e17072008-02-07 00:13:56 -080030#include <linux/swap.h>
31#include <linux/spinlock.h>
32#include <linux/fs.h>
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -080033#include <linux/seq_file.h>
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -070034#include <linux/vmalloc.h>
Christoph Lameterb69408e2008-10-18 20:26:14 -070035#include <linux/mm_inline.h>
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -070036#include <linux/page_cgroup.h>
Balbir Singh8cdea7c2008-02-07 00:13:50 -080037
Balbir Singh8697d332008-02-07 00:13:59 -080038#include <asm/uaccess.h>
39
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070040struct cgroup_subsys mem_cgroup_subsys __read_mostly;
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070041#define MEM_CGROUP_RECLAIM_RETRIES 5
Balbir Singh8cdea7c2008-02-07 00:13:50 -080042
43/*
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080044 * Statistics for memory cgroup.
45 */
46enum mem_cgroup_stat_index {
47 /*
48 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
49 */
50 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
51 MEM_CGROUP_STAT_RSS, /* # of pages charged as rss */
Balaji Rao55e462b2008-05-01 04:35:12 -070052 MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */
53 MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080054
55 MEM_CGROUP_STAT_NSTATS,
56};
57
58struct mem_cgroup_stat_cpu {
59 s64 count[MEM_CGROUP_STAT_NSTATS];
60} ____cacheline_aligned_in_smp;
61
62struct mem_cgroup_stat {
63 struct mem_cgroup_stat_cpu cpustat[NR_CPUS];
64};
65
66/*
67 * For accounting under irq disable, no need for increment preempt count.
68 */
KAMEZAWA Hiroyukiaddb9ef2008-10-18 20:28:10 -070069static inline void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat_cpu *stat,
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080070 enum mem_cgroup_stat_index idx, int val)
71{
KAMEZAWA Hiroyukiaddb9ef2008-10-18 20:28:10 -070072 stat->count[idx] += val;
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080073}
74
75static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
76 enum mem_cgroup_stat_index idx)
77{
78 int cpu;
79 s64 ret = 0;
80 for_each_possible_cpu(cpu)
81 ret += stat->cpustat[cpu].count[idx];
82 return ret;
83}
84
85/*
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -080086 * per-zone information in memory controller.
87 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -080088struct mem_cgroup_per_zone {
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -080089 /*
90 * spin_lock to protect the per cgroup LRU
91 */
92 spinlock_t lru_lock;
Christoph Lameterb69408e2008-10-18 20:26:14 -070093 struct list_head lists[NR_LRU_LISTS];
94 unsigned long count[NR_LRU_LISTS];
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -080095};
96/* Macro for accessing counter */
97#define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)])
98
99struct mem_cgroup_per_node {
100 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
101};
102
103struct mem_cgroup_lru_info {
104 struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
105};
106
107/*
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800108 * The memory controller data structure. The memory controller controls both
109 * page cache and RSS per cgroup. We would eventually like to provide
110 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
111 * to help the administrator determine what knobs to tune.
112 *
113 * TODO: Add a water mark for the memory controller. Reclaim will begin when
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800114 * we hit the water mark. May be even add a low water mark, such that
115 * no reclaim occurs from a cgroup at it's low water mark, this is
116 * a feature that will be implemented much later in the future.
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800117 */
118struct mem_cgroup {
119 struct cgroup_subsys_state css;
120 /*
121 * the counter to account for memory usage
122 */
123 struct res_counter res;
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800124 /*
125 * Per cgroup active and inactive list, similar to the
126 * per zone LRU lists.
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800127 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800128 struct mem_cgroup_lru_info info;
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800129
KAMEZAWA Hiroyuki6c48a1d2008-02-07 00:14:34 -0800130 int prev_priority; /* for recording reclaim priority */
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800131 /*
132 * statistics.
133 */
134 struct mem_cgroup_stat stat;
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800135};
Hugh Dickins8869b8f2008-03-04 14:29:09 -0800136static struct mem_cgroup init_mem_cgroup;
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800137
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800138enum charge_type {
139 MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
140 MEM_CGROUP_CHARGE_TYPE_MAPPED,
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700141 MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700142 MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */
143 NR_CHARGE_TYPE,
144};
145
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700146/* only for here (for easy reading.) */
147#define PCGF_CACHE (1UL << PCG_CACHE)
148#define PCGF_USED (1UL << PCG_USED)
149#define PCGF_ACTIVE (1UL << PCG_ACTIVE)
150#define PCGF_LOCK (1UL << PCG_LOCK)
151#define PCGF_FILE (1UL << PCG_FILE)
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700152static const unsigned long
153pcg_default_flags[NR_CHARGE_TYPE] = {
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700154 PCGF_CACHE | PCGF_FILE | PCGF_USED | PCGF_LOCK, /* File Cache */
155 PCGF_ACTIVE | PCGF_USED | PCGF_LOCK, /* Anon */
156 PCGF_ACTIVE | PCGF_CACHE | PCGF_USED | PCGF_LOCK, /* Shmem */
157 0, /* FORCE */
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800158};
159
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800160/*
161 * Always modified under lru lock. Then, not necessary to preempt_disable()
162 */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700163static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
164 struct page_cgroup *pc,
165 bool charge)
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800166{
167 int val = (charge)? 1 : -1;
168 struct mem_cgroup_stat *stat = &mem->stat;
KAMEZAWA Hiroyukiaddb9ef2008-10-18 20:28:10 -0700169 struct mem_cgroup_stat_cpu *cpustat;
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800170
Hugh Dickins8869b8f2008-03-04 14:29:09 -0800171 VM_BUG_ON(!irqs_disabled());
KAMEZAWA Hiroyukiaddb9ef2008-10-18 20:28:10 -0700172
173 cpustat = &stat->cpustat[smp_processor_id()];
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700174 if (PageCgroupCache(pc))
KAMEZAWA Hiroyukiaddb9ef2008-10-18 20:28:10 -0700175 __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_CACHE, val);
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800176 else
KAMEZAWA Hiroyukiaddb9ef2008-10-18 20:28:10 -0700177 __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_RSS, val);
Balaji Rao55e462b2008-05-01 04:35:12 -0700178
179 if (charge)
KAMEZAWA Hiroyukiaddb9ef2008-10-18 20:28:10 -0700180 __mem_cgroup_stat_add_safe(cpustat,
Balaji Rao55e462b2008-05-01 04:35:12 -0700181 MEM_CGROUP_STAT_PGPGIN_COUNT, 1);
182 else
KAMEZAWA Hiroyukiaddb9ef2008-10-18 20:28:10 -0700183 __mem_cgroup_stat_add_safe(cpustat,
Balaji Rao55e462b2008-05-01 04:35:12 -0700184 MEM_CGROUP_STAT_PGPGOUT_COUNT, 1);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800185}
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800186
Hugh Dickinsd5b69e32008-03-04 14:29:10 -0800187static struct mem_cgroup_per_zone *
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800188mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
189{
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800190 return &mem->info.nodeinfo[nid]->zoneinfo[zid];
191}
192
Hugh Dickinsd5b69e32008-03-04 14:29:10 -0800193static struct mem_cgroup_per_zone *
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800194page_cgroup_zoneinfo(struct page_cgroup *pc)
195{
196 struct mem_cgroup *mem = pc->mem_cgroup;
197 int nid = page_cgroup_nid(pc);
198 int zid = page_cgroup_zid(pc);
199
200 return mem_cgroup_zoneinfo(mem, nid, zid);
201}
202
203static unsigned long mem_cgroup_get_all_zonestat(struct mem_cgroup *mem,
Christoph Lameterb69408e2008-10-18 20:26:14 -0700204 enum lru_list idx)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800205{
206 int nid, zid;
207 struct mem_cgroup_per_zone *mz;
208 u64 total = 0;
209
210 for_each_online_node(nid)
211 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
212 mz = mem_cgroup_zoneinfo(mem, nid, zid);
213 total += MEM_CGROUP_ZSTAT(mz, idx);
214 }
215 return total;
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800216}
217
Hugh Dickinsd5b69e32008-03-04 14:29:10 -0800218static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800219{
220 return container_of(cgroup_subsys_state(cont,
221 mem_cgroup_subsys_id), struct mem_cgroup,
222 css);
223}
224
Balbir Singhcf475ad2008-04-29 01:00:16 -0700225struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800226{
Balbir Singh31a78f22008-09-28 23:09:31 +0100227 /*
228 * mm_update_next_owner() may clear mm->owner to NULL
229 * if it races with swapoff, page migration, etc.
230 * So this can be called with p == NULL.
231 */
232 if (unlikely(!p))
233 return NULL;
234
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800235 return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
236 struct mem_cgroup, css);
237}
238
KAMEZAWA Hiroyuki3eae90c2008-04-29 01:00:22 -0700239static void __mem_cgroup_remove_list(struct mem_cgroup_per_zone *mz,
240 struct page_cgroup *pc)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800241{
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700242 int lru = LRU_BASE;
243
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700244 if (PageCgroupUnevictable(pc))
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700245 lru = LRU_UNEVICTABLE;
246 else {
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700247 if (PageCgroupActive(pc))
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700248 lru += LRU_ACTIVE;
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700249 if (PageCgroupFile(pc))
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700250 lru += LRU_FILE;
251 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800252
Christoph Lameterb69408e2008-10-18 20:26:14 -0700253 MEM_CGROUP_ZSTAT(mz, lru) -= 1;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800254
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700255 mem_cgroup_charge_statistics(pc->mem_cgroup, pc, false);
KAMEZAWA Hiroyuki508b7be2008-07-25 01:47:09 -0700256 list_del(&pc->lru);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800257}
258
KAMEZAWA Hiroyuki3eae90c2008-04-29 01:00:22 -0700259static void __mem_cgroup_add_list(struct mem_cgroup_per_zone *mz,
260 struct page_cgroup *pc)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800261{
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700262 int lru = LRU_BASE;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800263
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700264 if (PageCgroupUnevictable(pc))
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700265 lru = LRU_UNEVICTABLE;
266 else {
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700267 if (PageCgroupActive(pc))
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700268 lru += LRU_ACTIVE;
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700269 if (PageCgroupFile(pc))
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700270 lru += LRU_FILE;
271 }
Christoph Lameterb69408e2008-10-18 20:26:14 -0700272
273 MEM_CGROUP_ZSTAT(mz, lru) += 1;
274 list_add(&pc->lru, &mz->lists[lru]);
275
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700276 mem_cgroup_charge_statistics(pc->mem_cgroup, pc, true);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800277}
278
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700279static void __mem_cgroup_move_lists(struct page_cgroup *pc, enum lru_list lru)
Balbir Singh66e17072008-02-07 00:13:56 -0800280{
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800281 struct mem_cgroup_per_zone *mz = page_cgroup_zoneinfo(pc);
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700282 int active = PageCgroupActive(pc);
283 int file = PageCgroupFile(pc);
284 int unevictable = PageCgroupUnevictable(pc);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700285 enum lru_list from = unevictable ? LRU_UNEVICTABLE :
286 (LRU_FILE * !!file + !!active);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800287
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700288 if (lru == from)
289 return;
Christoph Lameterb69408e2008-10-18 20:26:14 -0700290
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700291 MEM_CGROUP_ZSTAT(mz, from) -= 1;
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700292 /*
293 * However this is done under mz->lru_lock, another flags, which
294 * are not related to LRU, will be modified from out-of-lock.
295 * We have to use atomic set/clear flags.
296 */
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700297 if (is_unevictable_lru(lru)) {
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700298 ClearPageCgroupActive(pc);
299 SetPageCgroupUnevictable(pc);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700300 } else {
301 if (is_active_lru(lru))
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700302 SetPageCgroupActive(pc);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700303 else
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700304 ClearPageCgroupActive(pc);
305 ClearPageCgroupUnevictable(pc);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700306 }
Christoph Lameterb69408e2008-10-18 20:26:14 -0700307
Christoph Lameterb69408e2008-10-18 20:26:14 -0700308 MEM_CGROUP_ZSTAT(mz, lru) += 1;
309 list_move(&pc->lru, &mz->lists[lru]);
Balbir Singh66e17072008-02-07 00:13:56 -0800310}
311
David Rientjes4c4a2212008-02-07 00:14:06 -0800312int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
313{
314 int ret;
315
316 task_lock(task);
Hugh Dickinsbd845e32008-03-04 14:29:01 -0800317 ret = task->mm && mm_match_cgroup(task->mm, mem);
David Rientjes4c4a2212008-02-07 00:14:06 -0800318 task_unlock(task);
319 return ret;
320}
321
Balbir Singh66e17072008-02-07 00:13:56 -0800322/*
323 * This routine assumes that the appropriate zone's lru lock is already held
324 */
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700325void mem_cgroup_move_lists(struct page *page, enum lru_list lru)
Balbir Singh66e17072008-02-07 00:13:56 -0800326{
Hugh Dickins427d5412008-03-04 14:29:03 -0800327 struct page_cgroup *pc;
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800328 struct mem_cgroup_per_zone *mz;
329 unsigned long flags;
330
Li Zefancede86a2008-07-25 01:47:18 -0700331 if (mem_cgroup_subsys.disabled)
332 return;
333
Hugh Dickins2680eed2008-03-04 14:29:13 -0800334 /*
335 * We cannot lock_page_cgroup while holding zone's lru_lock,
336 * because other holders of lock_page_cgroup can be interrupted
337 * with an attempt to rotate_reclaimable_page. But we cannot
338 * safely get to page_cgroup without it, so just try_lock it:
339 * mem_cgroup_isolate_pages allows for page left on wrong list.
340 */
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700341 pc = lookup_page_cgroup(page);
342 if (!trylock_page_cgroup(pc))
Balbir Singh66e17072008-02-07 00:13:56 -0800343 return;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700344 if (pc && PageCgroupUsed(pc)) {
Hugh Dickins2680eed2008-03-04 14:29:13 -0800345 mz = page_cgroup_zoneinfo(pc);
Hugh Dickins2680eed2008-03-04 14:29:13 -0800346 spin_lock_irqsave(&mz->lru_lock, flags);
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700347 __mem_cgroup_move_lists(pc, lru);
Hugh Dickins2680eed2008-03-04 14:29:13 -0800348 spin_unlock_irqrestore(&mz->lru_lock, flags);
Hirokazu Takahashi9b3c0a02008-03-04 14:29:15 -0800349 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700350 unlock_page_cgroup(pc);
Balbir Singh66e17072008-02-07 00:13:56 -0800351}
352
KAMEZAWA Hiroyuki58ae83d2008-02-07 00:14:32 -0800353/*
354 * Calculate mapped_ratio under memory controller. This will be used in
355 * vmscan.c for deteremining we have to reclaim mapped pages.
356 */
357int mem_cgroup_calc_mapped_ratio(struct mem_cgroup *mem)
358{
359 long total, rss;
360
361 /*
362 * usage is recorded in bytes. But, here, we assume the number of
363 * physical pages can be represented by "long" on any arch.
364 */
365 total = (long) (mem->res.usage >> PAGE_SHIFT) + 1L;
366 rss = (long)mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS);
367 return (int)((rss * 100L) / total);
368}
Hugh Dickins8869b8f2008-03-04 14:29:09 -0800369
KAMEZAWA Hiroyuki5932f362008-02-07 00:14:33 -0800370/*
KAMEZAWA Hiroyuki6c48a1d2008-02-07 00:14:34 -0800371 * prev_priority control...this will be used in memory reclaim path.
372 */
373int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem)
374{
375 return mem->prev_priority;
376}
377
378void mem_cgroup_note_reclaim_priority(struct mem_cgroup *mem, int priority)
379{
380 if (priority < mem->prev_priority)
381 mem->prev_priority = priority;
382}
383
384void mem_cgroup_record_reclaim_priority(struct mem_cgroup *mem, int priority)
385{
386 mem->prev_priority = priority;
387}
388
KAMEZAWA Hiroyukicc381082008-02-07 00:14:35 -0800389/*
390 * Calculate # of pages to be scanned in this priority/zone.
391 * See also vmscan.c
392 *
393 * priority starts from "DEF_PRIORITY" and decremented in each loop.
394 * (see include/linux/mmzone.h)
395 */
396
Christoph Lameterb69408e2008-10-18 20:26:14 -0700397long mem_cgroup_calc_reclaim(struct mem_cgroup *mem, struct zone *zone,
398 int priority, enum lru_list lru)
KAMEZAWA Hiroyukicc381082008-02-07 00:14:35 -0800399{
Christoph Lameterb69408e2008-10-18 20:26:14 -0700400 long nr_pages;
KAMEZAWA Hiroyukicc381082008-02-07 00:14:35 -0800401 int nid = zone->zone_pgdat->node_id;
402 int zid = zone_idx(zone);
403 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);
404
Christoph Lameterb69408e2008-10-18 20:26:14 -0700405 nr_pages = MEM_CGROUP_ZSTAT(mz, lru);
KAMEZAWA Hiroyukicc381082008-02-07 00:14:35 -0800406
Christoph Lameterb69408e2008-10-18 20:26:14 -0700407 return (nr_pages >> priority);
KAMEZAWA Hiroyukicc381082008-02-07 00:14:35 -0800408}
409
Balbir Singh66e17072008-02-07 00:13:56 -0800410unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
411 struct list_head *dst,
412 unsigned long *scanned, int order,
413 int mode, struct zone *z,
414 struct mem_cgroup *mem_cont,
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700415 int active, int file)
Balbir Singh66e17072008-02-07 00:13:56 -0800416{
417 unsigned long nr_taken = 0;
418 struct page *page;
419 unsigned long scan;
420 LIST_HEAD(pc_list);
421 struct list_head *src;
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -0800422 struct page_cgroup *pc, *tmp;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -0800423 int nid = z->zone_pgdat->node_id;
424 int zid = zone_idx(z);
425 struct mem_cgroup_per_zone *mz;
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700426 int lru = LRU_FILE * !!file + !!active;
Balbir Singh66e17072008-02-07 00:13:56 -0800427
Balbir Singhcf475ad2008-04-29 01:00:16 -0700428 BUG_ON(!mem_cont);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -0800429 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
Christoph Lameterb69408e2008-10-18 20:26:14 -0700430 src = &mz->lists[lru];
Balbir Singh66e17072008-02-07 00:13:56 -0800431
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800432 spin_lock(&mz->lru_lock);
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -0800433 scan = 0;
434 list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
Hugh Dickins436c65412008-02-07 00:14:12 -0800435 if (scan >= nr_to_scan)
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -0800436 break;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700437 if (unlikely(!PageCgroupUsed(pc)))
438 continue;
Balbir Singh66e17072008-02-07 00:13:56 -0800439 page = pc->page;
Balbir Singh66e17072008-02-07 00:13:56 -0800440
Hugh Dickins436c65412008-02-07 00:14:12 -0800441 if (unlikely(!PageLRU(page)))
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -0800442 continue;
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -0800443
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700444 /*
445 * TODO: play better with lumpy reclaim, grabbing anything.
446 */
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700447 if (PageUnevictable(page) ||
448 (PageActive(page) && !active) ||
449 (!PageActive(page) && active)) {
450 __mem_cgroup_move_lists(pc, page_lru(page));
Balbir Singh66e17072008-02-07 00:13:56 -0800451 continue;
452 }
453
Hugh Dickins436c65412008-02-07 00:14:12 -0800454 scan++;
455 list_move(&pc->lru, &pc_list);
Balbir Singh66e17072008-02-07 00:13:56 -0800456
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700457 if (__isolate_lru_page(page, mode, file) == 0) {
Balbir Singh66e17072008-02-07 00:13:56 -0800458 list_move(&page->lru, dst);
459 nr_taken++;
460 }
461 }
462
463 list_splice(&pc_list, src);
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800464 spin_unlock(&mz->lru_lock);
Balbir Singh66e17072008-02-07 00:13:56 -0800465
466 *scanned = scan;
467 return nr_taken;
468}
469
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800470/*
471 * Charge the memory controller for page usage.
472 * Return
473 * 0 if the charge was successful
474 * < 0 if the cgroup is over its limit
475 */
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800476static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700477 gfp_t gfp_mask, enum charge_type ctype,
478 struct mem_cgroup *memcg)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800479{
480 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki9175e032008-02-07 00:14:08 -0800481 struct page_cgroup *pc;
Balbir Singh66e17072008-02-07 00:13:56 -0800482 unsigned long nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800483 struct mem_cgroup_per_zone *mz;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700484 unsigned long flags;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800485
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700486 pc = lookup_page_cgroup(page);
487 /* can happen at boot */
488 if (unlikely(!pc))
489 return 0;
490 prefetchw(pc);
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800491 /*
Hugh Dickins3be91272008-02-07 00:14:19 -0800492 * We always charge the cgroup the mm_struct belongs to.
493 * The mm_struct's mem_cgroup changes on task migration if the
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800494 * thread group leader migrates. It's possible that mm is not
495 * set, if so charge the init_mm (happens for pagecache usage).
496 */
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700497
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700498 if (likely(!memcg)) {
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700499 rcu_read_lock();
500 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
Balbir Singh31a78f22008-09-28 23:09:31 +0100501 if (unlikely(!mem)) {
502 rcu_read_unlock();
Balbir Singh31a78f22008-09-28 23:09:31 +0100503 return 0;
504 }
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700505 /*
506 * For every charge from the cgroup, increment reference count
507 */
508 css_get(&mem->css);
509 rcu_read_unlock();
510 } else {
511 mem = memcg;
512 css_get(&memcg->css);
513 }
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800514
KAMEZAWA Hiroyukiaddb9ef2008-10-18 20:28:10 -0700515 while (unlikely(res_counter_charge(&mem->res, PAGE_SIZE))) {
Hugh Dickins3be91272008-02-07 00:14:19 -0800516 if (!(gfp_mask & __GFP_WAIT))
517 goto out;
Balbir Singhe1a1cd52008-02-07 00:14:02 -0800518
519 if (try_to_free_mem_cgroup_pages(mem, gfp_mask))
Balbir Singh66e17072008-02-07 00:13:56 -0800520 continue;
521
522 /*
Hugh Dickins8869b8f2008-03-04 14:29:09 -0800523 * try_to_free_mem_cgroup_pages() might not give us a full
524 * picture of reclaim. Some pages are reclaimed and might be
525 * moved to swap cache or just unmapped from the cgroup.
526 * Check the limit again to see if the reclaim reduced the
527 * current usage of the cgroup before giving up
528 */
Balbir Singh66e17072008-02-07 00:13:56 -0800529 if (res_counter_check_under_limit(&mem->res))
530 continue;
Hugh Dickins3be91272008-02-07 00:14:19 -0800531
532 if (!nr_retries--) {
533 mem_cgroup_out_of_memory(mem, gfp_mask);
534 goto out;
Balbir Singh66e17072008-02-07 00:13:56 -0800535 }
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800536 }
537
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700538
539 lock_page_cgroup(pc);
540 if (unlikely(PageCgroupUsed(pc))) {
541 unlock_page_cgroup(pc);
542 res_counter_uncharge(&mem->res, PAGE_SIZE);
543 css_put(&mem->css);
544
545 goto done;
546 }
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800547 pc->mem_cgroup = mem;
KAMEZAWA Hiroyuki508b7be2008-07-25 01:47:09 -0700548 /*
549 * If a page is accounted as a page cache, insert to inactive list.
550 * If anon, insert to active list.
551 */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700552 pc->flags = pcg_default_flags[ctype];
Hugh Dickins3be91272008-02-07 00:14:19 -0800553
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800554 mz = page_cgroup_zoneinfo(pc);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700555
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800556 spin_lock_irqsave(&mz->lru_lock, flags);
KAMEZAWA Hiroyuki3eae90c2008-04-29 01:00:22 -0700557 __mem_cgroup_add_list(mz, pc);
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800558 spin_unlock_irqrestore(&mz->lru_lock, flags);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700559 unlock_page_cgroup(pc);
Balbir Singh66e17072008-02-07 00:13:56 -0800560
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800561done:
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800562 return 0;
Hugh Dickins3be91272008-02-07 00:14:19 -0800563out:
564 css_put(&mem->css);
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800565 return -ENOMEM;
566}
567
Hugh Dickins8869b8f2008-03-04 14:29:09 -0800568int mem_cgroup_charge(struct page *page, struct mm_struct *mm, gfp_t gfp_mask)
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800569{
Li Zefancede86a2008-07-25 01:47:18 -0700570 if (mem_cgroup_subsys.disabled)
571 return 0;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700572 if (PageCompound(page))
573 return 0;
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700574 /*
575 * If already mapped, we don't have to account.
576 * If page cache, page->mapping has address_space.
577 * But page->mapping may have out-of-use anon_vma pointer,
578 * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
579 * is NULL.
580 */
581 if (page_mapped(page) || (page->mapping && !PageAnon(page)))
582 return 0;
583 if (unlikely(!mm))
584 mm = &init_mm;
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800585 return mem_cgroup_charge_common(page, mm, gfp_mask,
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700586 MEM_CGROUP_CHARGE_TYPE_MAPPED, NULL);
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800587}
588
Balbir Singhe1a1cd52008-02-07 00:14:02 -0800589int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
590 gfp_t gfp_mask)
Balbir Singh8697d332008-02-07 00:13:59 -0800591{
Li Zefancede86a2008-07-25 01:47:18 -0700592 if (mem_cgroup_subsys.disabled)
593 return 0;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700594 if (PageCompound(page))
595 return 0;
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -0700596 /*
597 * Corner case handling. This is called from add_to_page_cache()
598 * in usual. But some FS (shmem) precharges this page before calling it
599 * and call add_to_page_cache() with GFP_NOWAIT.
600 *
601 * For GFP_NOWAIT case, the page may be pre-charged before calling
602 * add_to_page_cache(). (See shmem.c) check it here and avoid to call
603 * charge twice. (It works but has to pay a bit larger cost.)
604 */
605 if (!(gfp_mask & __GFP_WAIT)) {
606 struct page_cgroup *pc;
607
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700608
609 pc = lookup_page_cgroup(page);
610 if (!pc)
611 return 0;
612 lock_page_cgroup(pc);
613 if (PageCgroupUsed(pc)) {
614 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -0700615 return 0;
616 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700617 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -0700618 }
619
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700620 if (unlikely(!mm))
Balbir Singh8697d332008-02-07 00:13:59 -0800621 mm = &init_mm;
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -0700622
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700623 if (page_is_file_cache(page))
624 return mem_cgroup_charge_common(page, mm, gfp_mask,
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700625 MEM_CGROUP_CHARGE_TYPE_CACHE, NULL);
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700626 else
627 return mem_cgroup_charge_common(page, mm, gfp_mask,
628 MEM_CGROUP_CHARGE_TYPE_SHMEM, NULL);
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700629}
630
Balbir Singh8697d332008-02-07 00:13:59 -0800631/*
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700632 * uncharge if !page_mapped(page)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800633 */
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700634static void
635__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800636{
Hugh Dickins82895462008-03-04 14:29:08 -0800637 struct page_cgroup *pc;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800638 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800639 struct mem_cgroup_per_zone *mz;
Balbir Singh66e17072008-02-07 00:13:56 -0800640 unsigned long flags;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800641
Balbir Singh40779602008-04-04 14:29:59 -0700642 if (mem_cgroup_subsys.disabled)
643 return;
644
Balbir Singh8697d332008-02-07 00:13:59 -0800645 /*
Balbir Singh3c541e12008-02-07 00:14:41 -0800646 * Check if our page_cgroup is valid
Balbir Singh8697d332008-02-07 00:13:59 -0800647 */
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700648 pc = lookup_page_cgroup(page);
649 if (unlikely(!pc || !PageCgroupUsed(pc)))
650 return;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800651
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700652 lock_page_cgroup(pc);
653 if ((ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED && page_mapped(page))
654 || !PageCgroupUsed(pc)) {
655 /* This happens at race in zap_pte_range() and do_swap_page()*/
656 unlock_page_cgroup(pc);
657 return;
658 }
659 ClearPageCgroupUsed(pc);
660 mem = pc->mem_cgroup;
Hugh Dickinsb9c565d2008-03-04 14:29:11 -0800661
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700662 mz = page_cgroup_zoneinfo(pc);
663 spin_lock_irqsave(&mz->lru_lock, flags);
664 __mem_cgroup_remove_list(mz, pc);
665 spin_unlock_irqrestore(&mz->lru_lock, flags);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700666 unlock_page_cgroup(pc);
Hugh Dickinsfb59e9f2008-03-04 14:29:16 -0800667
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700668 res_counter_uncharge(&mem->res, PAGE_SIZE);
669 css_put(&mem->css);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800670
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700671 return;
Balbir Singh3c541e12008-02-07 00:14:41 -0800672}
673
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700674void mem_cgroup_uncharge_page(struct page *page)
675{
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700676 /* early check. */
677 if (page_mapped(page))
678 return;
679 if (page->mapping && !PageAnon(page))
680 return;
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700681 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
682}
683
684void mem_cgroup_uncharge_cache_page(struct page *page)
685{
686 VM_BUG_ON(page_mapped(page));
KAMEZAWA Hiroyukib7abea92008-10-18 20:28:09 -0700687 VM_BUG_ON(page->mapping);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700688 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
689}
690
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -0800691/*
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700692 * Before starting migration, account against new page.
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -0800693 */
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700694int mem_cgroup_prepare_migration(struct page *page, struct page *newpage)
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -0800695{
696 struct page_cgroup *pc;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700697 struct mem_cgroup *mem = NULL;
698 enum charge_type ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
699 int ret = 0;
Hugh Dickins8869b8f2008-03-04 14:29:09 -0800700
Balbir Singh40779602008-04-04 14:29:59 -0700701 if (mem_cgroup_subsys.disabled)
702 return 0;
703
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700704 pc = lookup_page_cgroup(page);
705 lock_page_cgroup(pc);
706 if (PageCgroupUsed(pc)) {
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700707 mem = pc->mem_cgroup;
708 css_get(&mem->css);
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700709 if (PageCgroupCache(pc)) {
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700710 if (page_is_file_cache(page))
711 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
712 else
713 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
714 }
Hugh Dickinsb9c565d2008-03-04 14:29:11 -0800715 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700716 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700717 if (mem) {
718 ret = mem_cgroup_charge_common(newpage, NULL, GFP_KERNEL,
719 ctype, mem);
720 css_put(&mem->css);
721 }
722 return ret;
723}
Hugh Dickinsfb59e9f2008-03-04 14:29:16 -0800724
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700725/* remove redundant charge if migration failed*/
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700726void mem_cgroup_end_migration(struct page *newpage)
727{
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700728 /*
729 * At success, page->mapping is not NULL.
730 * special rollback care is necessary when
731 * 1. at migration failure. (newpage->mapping is cleared in this case)
732 * 2. the newpage was moved but not remapped again because the task
733 * exits and the newpage is obsolete. In this case, the new page
734 * may be a swapcache. So, we just call mem_cgroup_uncharge_page()
735 * always for avoiding mess. The page_cgroup will be removed if
736 * unnecessary. File cache pages is still on radix-tree. Don't
737 * care it.
738 */
739 if (!newpage->mapping)
740 __mem_cgroup_uncharge_common(newpage,
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700741 MEM_CGROUP_CHARGE_TYPE_FORCE);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700742 else if (PageAnon(newpage))
743 mem_cgroup_uncharge_page(newpage);
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -0800744}
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800745
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800746/*
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -0700747 * A call to try to shrink memory usage under specified resource controller.
748 * This is typically used for page reclaiming for shmem for reducing side
749 * effect of page allocation from shmem, which is used by some mem_cgroup.
750 */
751int mem_cgroup_shrink_usage(struct mm_struct *mm, gfp_t gfp_mask)
752{
753 struct mem_cgroup *mem;
754 int progress = 0;
755 int retry = MEM_CGROUP_RECLAIM_RETRIES;
756
Li Zefancede86a2008-07-25 01:47:18 -0700757 if (mem_cgroup_subsys.disabled)
758 return 0;
Hugh Dickins9623e072008-08-12 15:08:41 -0700759 if (!mm)
760 return 0;
Li Zefancede86a2008-07-25 01:47:18 -0700761
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -0700762 rcu_read_lock();
763 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
Balbir Singh31a78f22008-09-28 23:09:31 +0100764 if (unlikely(!mem)) {
765 rcu_read_unlock();
766 return 0;
767 }
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -0700768 css_get(&mem->css);
769 rcu_read_unlock();
770
771 do {
772 progress = try_to_free_mem_cgroup_pages(mem, gfp_mask);
Daisuke Nishimuraa10cebf2008-09-22 13:57:52 -0700773 progress += res_counter_check_under_limit(&mem->res);
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -0700774 } while (!progress && --retry);
775
776 css_put(&mem->css);
777 if (!retry)
778 return -ENOMEM;
779 return 0;
780}
781
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -0700782int mem_cgroup_resize_limit(struct mem_cgroup *memcg, unsigned long long val)
783{
784
785 int retry_count = MEM_CGROUP_RECLAIM_RETRIES;
786 int progress;
787 int ret = 0;
788
789 while (res_counter_set_limit(&memcg->res, val)) {
790 if (signal_pending(current)) {
791 ret = -EINTR;
792 break;
793 }
794 if (!retry_count) {
795 ret = -EBUSY;
796 break;
797 }
798 progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL);
799 if (!progress)
800 retry_count--;
801 }
802 return ret;
803}
804
805
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -0700806/*
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800807 * This routine traverse page_cgroup in given list and drop them all.
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800808 * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
809 */
810#define FORCE_UNCHARGE_BATCH (128)
Hugh Dickins8869b8f2008-03-04 14:29:09 -0800811static void mem_cgroup_force_empty_list(struct mem_cgroup *mem,
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800812 struct mem_cgroup_per_zone *mz,
Christoph Lameterb69408e2008-10-18 20:26:14 -0700813 enum lru_list lru)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800814{
815 struct page_cgroup *pc;
816 struct page *page;
Hirokazu Takahashi9b3c0a02008-03-04 14:29:15 -0800817 int count = FORCE_UNCHARGE_BATCH;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800818 unsigned long flags;
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800819 struct list_head *list;
820
Christoph Lameterb69408e2008-10-18 20:26:14 -0700821 list = &mz->lists[lru];
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800822
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800823 spin_lock_irqsave(&mz->lru_lock, flags);
Hirokazu Takahashi9b3c0a02008-03-04 14:29:15 -0800824 while (!list_empty(list)) {
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800825 pc = list_entry(list->prev, struct page_cgroup, lru);
826 page = pc->page;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700827 if (!PageCgroupUsed(pc))
828 break;
Hirokazu Takahashi9b3c0a02008-03-04 14:29:15 -0800829 get_page(page);
830 spin_unlock_irqrestore(&mz->lru_lock, flags);
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700831 /*
832 * Check if this page is on LRU. !LRU page can be found
833 * if it's under page migration.
834 */
835 if (PageLRU(page)) {
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -0700836 __mem_cgroup_uncharge_common(page,
837 MEM_CGROUP_CHARGE_TYPE_FORCE);
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -0700838 put_page(page);
839 if (--count <= 0) {
840 count = FORCE_UNCHARGE_BATCH;
841 cond_resched();
842 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700843 } else {
844 spin_lock_irqsave(&mz->lru_lock, flags);
845 break;
846 }
Hirokazu Takahashi9b3c0a02008-03-04 14:29:15 -0800847 spin_lock_irqsave(&mz->lru_lock, flags);
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800848 }
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800849 spin_unlock_irqrestore(&mz->lru_lock, flags);
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800850}
851
852/*
853 * make mem_cgroup's charge to be 0 if there is no task.
854 * This enables deleting this mem_cgroup.
855 */
Hugh Dickinsd5b69e32008-03-04 14:29:10 -0800856static int mem_cgroup_force_empty(struct mem_cgroup *mem)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800857{
858 int ret = -EBUSY;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -0800859 int node, zid;
Hugh Dickins8869b8f2008-03-04 14:29:09 -0800860
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800861 css_get(&mem->css);
862 /*
863 * page reclaim code (kswapd etc..) will move pages between
Hugh Dickins8869b8f2008-03-04 14:29:09 -0800864 * active_list <-> inactive_list while we don't take a lock.
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800865 * So, we have to do loop here until all lists are empty.
866 */
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -0800867 while (mem->res.usage > 0) {
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800868 if (atomic_read(&mem->css.cgroup->count) > 0)
869 goto out;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700870 /* This is for making all *used* pages to be on LRU. */
871 lru_add_drain_all();
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -0800872 for_each_node_state(node, N_POSSIBLE)
873 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
874 struct mem_cgroup_per_zone *mz;
Christoph Lameterb69408e2008-10-18 20:26:14 -0700875 enum lru_list l;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -0800876 mz = mem_cgroup_zoneinfo(mem, node, zid);
Christoph Lameterb69408e2008-10-18 20:26:14 -0700877 for_each_lru(l)
878 mem_cgroup_force_empty_list(mem, mz, l);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -0800879 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700880 cond_resched();
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800881 }
882 ret = 0;
883out:
884 css_put(&mem->css);
885 return ret;
886}
887
Paul Menage2c3daa72008-04-29 00:59:58 -0700888static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800889{
Paul Menage2c3daa72008-04-29 00:59:58 -0700890 return res_counter_read_u64(&mem_cgroup_from_cont(cont)->res,
891 cft->private);
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800892}
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -0700893/*
894 * The user of this function is...
895 * RES_LIMIT.
896 */
Paul Menage856c13a2008-07-25 01:47:04 -0700897static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
898 const char *buffer)
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800899{
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -0700900 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
901 unsigned long long val;
902 int ret;
903
904 switch (cft->private) {
905 case RES_LIMIT:
906 /* This function does all necessary parse...reuse it */
907 ret = res_counter_memparse_write_strategy(buffer, &val);
908 if (!ret)
909 ret = mem_cgroup_resize_limit(memcg, val);
910 break;
911 default:
912 ret = -EINVAL; /* should be BUG() ? */
913 break;
914 }
915 return ret;
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800916}
917
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -0700918static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
Pavel Emelyanovc84872e2008-04-29 01:00:17 -0700919{
920 struct mem_cgroup *mem;
921
922 mem = mem_cgroup_from_cont(cont);
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -0700923 switch (event) {
924 case RES_MAX_USAGE:
925 res_counter_reset_max(&mem->res);
926 break;
927 case RES_FAILCNT:
928 res_counter_reset_failcnt(&mem->res);
929 break;
930 }
Pavel Emelyanov85cc59d2008-04-29 01:00:20 -0700931 return 0;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -0700932}
933
Pavel Emelyanov85cc59d2008-04-29 01:00:20 -0700934static int mem_force_empty_write(struct cgroup *cont, unsigned int event)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800935{
Pavel Emelyanov85cc59d2008-04-29 01:00:20 -0700936 return mem_cgroup_force_empty(mem_cgroup_from_cont(cont));
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -0800937}
938
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -0800939static const struct mem_cgroup_stat_desc {
940 const char *msg;
941 u64 unit;
942} mem_cgroup_stat_desc[] = {
943 [MEM_CGROUP_STAT_CACHE] = { "cache", PAGE_SIZE, },
944 [MEM_CGROUP_STAT_RSS] = { "rss", PAGE_SIZE, },
Balaji Rao55e462b2008-05-01 04:35:12 -0700945 [MEM_CGROUP_STAT_PGPGIN_COUNT] = {"pgpgin", 1, },
946 [MEM_CGROUP_STAT_PGPGOUT_COUNT] = {"pgpgout", 1, },
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -0800947};
948
Paul Menagec64745c2008-04-29 01:00:02 -0700949static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
950 struct cgroup_map_cb *cb)
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -0800951{
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -0800952 struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
953 struct mem_cgroup_stat *stat = &mem_cont->stat;
954 int i;
955
956 for (i = 0; i < ARRAY_SIZE(stat->cpustat[0].count); i++) {
957 s64 val;
958
959 val = mem_cgroup_read_stat(stat, i);
960 val *= mem_cgroup_stat_desc[i].unit;
Paul Menagec64745c2008-04-29 01:00:02 -0700961 cb->fill(cb, mem_cgroup_stat_desc[i].msg, val);
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -0800962 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800963 /* showing # of active pages */
964 {
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700965 unsigned long active_anon, inactive_anon;
966 unsigned long active_file, inactive_file;
Lee Schermerhorn7b854122008-10-18 20:26:40 -0700967 unsigned long unevictable;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800968
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700969 inactive_anon = mem_cgroup_get_all_zonestat(mem_cont,
970 LRU_INACTIVE_ANON);
971 active_anon = mem_cgroup_get_all_zonestat(mem_cont,
972 LRU_ACTIVE_ANON);
973 inactive_file = mem_cgroup_get_all_zonestat(mem_cont,
974 LRU_INACTIVE_FILE);
975 active_file = mem_cgroup_get_all_zonestat(mem_cont,
976 LRU_ACTIVE_FILE);
Lee Schermerhorn7b854122008-10-18 20:26:40 -0700977 unevictable = mem_cgroup_get_all_zonestat(mem_cont,
978 LRU_UNEVICTABLE);
979
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700980 cb->fill(cb, "active_anon", (active_anon) * PAGE_SIZE);
981 cb->fill(cb, "inactive_anon", (inactive_anon) * PAGE_SIZE);
982 cb->fill(cb, "active_file", (active_file) * PAGE_SIZE);
983 cb->fill(cb, "inactive_file", (inactive_file) * PAGE_SIZE);
Lee Schermerhorn7b854122008-10-18 20:26:40 -0700984 cb->fill(cb, "unevictable", unevictable * PAGE_SIZE);
985
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800986 }
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -0800987 return 0;
988}
989
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800990static struct cftype mem_cgroup_files[] = {
991 {
Balbir Singh0eea1032008-02-07 00:13:57 -0800992 .name = "usage_in_bytes",
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800993 .private = RES_USAGE,
Paul Menage2c3daa72008-04-29 00:59:58 -0700994 .read_u64 = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800995 },
996 {
Pavel Emelyanovc84872e2008-04-29 01:00:17 -0700997 .name = "max_usage_in_bytes",
998 .private = RES_MAX_USAGE,
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -0700999 .trigger = mem_cgroup_reset,
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07001000 .read_u64 = mem_cgroup_read,
1001 },
1002 {
Balbir Singh0eea1032008-02-07 00:13:57 -08001003 .name = "limit_in_bytes",
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001004 .private = RES_LIMIT,
Paul Menage856c13a2008-07-25 01:47:04 -07001005 .write_string = mem_cgroup_write,
Paul Menage2c3daa72008-04-29 00:59:58 -07001006 .read_u64 = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001007 },
1008 {
1009 .name = "failcnt",
1010 .private = RES_FAILCNT,
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07001011 .trigger = mem_cgroup_reset,
Paul Menage2c3daa72008-04-29 00:59:58 -07001012 .read_u64 = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001013 },
Balbir Singh8697d332008-02-07 00:13:59 -08001014 {
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08001015 .name = "force_empty",
Pavel Emelyanov85cc59d2008-04-29 01:00:20 -07001016 .trigger = mem_force_empty_write,
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08001017 },
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08001018 {
1019 .name = "stat",
Paul Menagec64745c2008-04-29 01:00:02 -07001020 .read_map = mem_control_stat_show,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08001021 },
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001022};
1023
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001024static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
1025{
1026 struct mem_cgroup_per_node *pn;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001027 struct mem_cgroup_per_zone *mz;
Christoph Lameterb69408e2008-10-18 20:26:14 -07001028 enum lru_list l;
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07001029 int zone, tmp = node;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001030 /*
1031 * This routine is called against possible nodes.
1032 * But it's BUG to call kmalloc() against offline node.
1033 *
1034 * TODO: this routine can waste much memory for nodes which will
1035 * never be onlined. It's better to use memory hotplug callback
1036 * function.
1037 */
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07001038 if (!node_state(node, N_NORMAL_MEMORY))
1039 tmp = -1;
1040 pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001041 if (!pn)
1042 return 1;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001043
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001044 mem->info.nodeinfo[node] = pn;
1045 memset(pn, 0, sizeof(*pn));
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001046
1047 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
1048 mz = &pn->zoneinfo[zone];
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -08001049 spin_lock_init(&mz->lru_lock);
Christoph Lameterb69408e2008-10-18 20:26:14 -07001050 for_each_lru(l)
1051 INIT_LIST_HEAD(&mz->lists[l]);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001052 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001053 return 0;
1054}
1055
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001056static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
1057{
1058 kfree(mem->info.nodeinfo[node]);
1059}
1060
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07001061static struct mem_cgroup *mem_cgroup_alloc(void)
1062{
1063 struct mem_cgroup *mem;
1064
1065 if (sizeof(*mem) < PAGE_SIZE)
1066 mem = kmalloc(sizeof(*mem), GFP_KERNEL);
1067 else
1068 mem = vmalloc(sizeof(*mem));
1069
1070 if (mem)
1071 memset(mem, 0, sizeof(*mem));
1072 return mem;
1073}
1074
1075static void mem_cgroup_free(struct mem_cgroup *mem)
1076{
1077 if (sizeof(*mem) < PAGE_SIZE)
1078 kfree(mem);
1079 else
1080 vfree(mem);
1081}
1082
1083
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001084static struct cgroup_subsys_state *
1085mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
1086{
1087 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001088 int node;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001089
Balbir Singhb6ac57d2008-04-29 01:00:19 -07001090 if (unlikely((cont->parent) == NULL)) {
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001091 mem = &init_mem_cgroup;
Balbir Singhb6ac57d2008-04-29 01:00:19 -07001092 } else {
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07001093 mem = mem_cgroup_alloc();
1094 if (!mem)
1095 return ERR_PTR(-ENOMEM);
Balbir Singhb6ac57d2008-04-29 01:00:19 -07001096 }
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001097
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001098 res_counter_init(&mem->res);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001099
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001100 for_each_node_state(node, N_POSSIBLE)
1101 if (alloc_mem_cgroup_per_zone_info(mem, node))
1102 goto free_out;
1103
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001104 return &mem->css;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001105free_out:
1106 for_each_node_state(node, N_POSSIBLE)
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001107 free_mem_cgroup_per_zone_info(mem, node);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001108 if (cont->parent != NULL)
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07001109 mem_cgroup_free(mem);
Li Zefan2dda81c2008-02-23 15:24:14 -08001110 return ERR_PTR(-ENOMEM);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001111}
1112
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08001113static void mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
1114 struct cgroup *cont)
1115{
1116 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
1117 mem_cgroup_force_empty(mem);
1118}
1119
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001120static void mem_cgroup_destroy(struct cgroup_subsys *ss,
1121 struct cgroup *cont)
1122{
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001123 int node;
1124 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
1125
1126 for_each_node_state(node, N_POSSIBLE)
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001127 free_mem_cgroup_per_zone_info(mem, node);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001128
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07001129 mem_cgroup_free(mem_cgroup_from_cont(cont));
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001130}
1131
1132static int mem_cgroup_populate(struct cgroup_subsys *ss,
1133 struct cgroup *cont)
1134{
1135 return cgroup_add_files(cont, ss, mem_cgroup_files,
1136 ARRAY_SIZE(mem_cgroup_files));
1137}
1138
Balbir Singh67e465a2008-02-07 00:13:54 -08001139static void mem_cgroup_move_task(struct cgroup_subsys *ss,
1140 struct cgroup *cont,
1141 struct cgroup *old_cont,
1142 struct task_struct *p)
1143{
1144 struct mm_struct *mm;
1145 struct mem_cgroup *mem, *old_mem;
1146
1147 mm = get_task_mm(p);
1148 if (mm == NULL)
1149 return;
1150
1151 mem = mem_cgroup_from_cont(cont);
1152 old_mem = mem_cgroup_from_cont(old_cont);
1153
Balbir Singh67e465a2008-02-07 00:13:54 -08001154 /*
1155 * Only thread group leaders are allowed to migrate, the mm_struct is
1156 * in effect owned by the leader
1157 */
Pavel Emelyanov52ea27e2008-03-19 17:00:45 -07001158 if (!thread_group_leader(p))
Balbir Singh67e465a2008-02-07 00:13:54 -08001159 goto out;
1160
Balbir Singh67e465a2008-02-07 00:13:54 -08001161out:
1162 mmput(mm);
Balbir Singh67e465a2008-02-07 00:13:54 -08001163}
1164
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001165struct cgroup_subsys mem_cgroup_subsys = {
1166 .name = "memory",
1167 .subsys_id = mem_cgroup_subsys_id,
1168 .create = mem_cgroup_create,
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08001169 .pre_destroy = mem_cgroup_pre_destroy,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001170 .destroy = mem_cgroup_destroy,
1171 .populate = mem_cgroup_populate,
Balbir Singh67e465a2008-02-07 00:13:54 -08001172 .attach = mem_cgroup_move_task,
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001173 .early_init = 0,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001174};