blob: 3eed583895a6f31eb434a252697371c7daa6bbd4 [file] [log] [blame]
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001/* memcontrol.c - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08006 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08009 * Memory thresholds
10 * Copyright (C) 2009 Nokia Corporation
11 * Author: Kirill A. Shutemov
12 *
Balbir Singh8cdea7c2008-02-07 00:13:50 -080013 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 */
23
24#include <linux/res_counter.h>
25#include <linux/memcontrol.h>
26#include <linux/cgroup.h>
Pavel Emelianov78fb7462008-02-07 00:13:51 -080027#include <linux/mm.h>
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -080028#include <linux/hugetlb.h>
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -080029#include <linux/pagemap.h>
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080030#include <linux/smp.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080031#include <linux/page-flags.h>
Balbir Singh66e17072008-02-07 00:13:56 -080032#include <linux/backing-dev.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080033#include <linux/bit_spinlock.h>
34#include <linux/rcupdate.h>
Balbir Singhe2224322009-04-02 16:57:39 -070035#include <linux/limits.h>
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -080036#include <linux/mutex.h>
Balbir Singhf64c3f52009-09-23 15:56:37 -070037#include <linux/rbtree.h>
Balbir Singhb6ac57d2008-04-29 01:00:19 -070038#include <linux/slab.h>
Balbir Singh66e17072008-02-07 00:13:56 -080039#include <linux/swap.h>
Daisuke Nishimura02491442010-03-10 15:22:17 -080040#include <linux/swapops.h>
Balbir Singh66e17072008-02-07 00:13:56 -080041#include <linux/spinlock.h>
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080042#include <linux/eventfd.h>
43#include <linux/sort.h>
Balbir Singh66e17072008-02-07 00:13:56 -080044#include <linux/fs.h>
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -080045#include <linux/seq_file.h>
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -070046#include <linux/vmalloc.h>
Christoph Lameterb69408e2008-10-18 20:26:14 -070047#include <linux/mm_inline.h>
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -070048#include <linux/page_cgroup.h>
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -080049#include <linux/cpu.h>
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -070050#include <linux/oom.h>
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -080051#include "internal.h"
Balbir Singh8cdea7c2008-02-07 00:13:50 -080052
Balbir Singh8697d332008-02-07 00:13:59 -080053#include <asm/uaccess.h>
54
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -070055#include <trace/events/vmscan.h>
56
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070057struct cgroup_subsys mem_cgroup_subsys __read_mostly;
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070058#define MEM_CGROUP_RECLAIM_RETRIES 5
Balbir Singh4b3bde42009-09-23 15:56:32 -070059struct mem_cgroup *root_mem_cgroup __read_mostly;
Balbir Singh8cdea7c2008-02-07 00:13:50 -080060
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080061#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
Li Zefan338c8432009-06-17 16:27:15 -070062/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080063int do_swap_account __read_mostly;
64static int really_do_swap_account __initdata = 1; /* for remember boot option*/
65#else
66#define do_swap_account (0)
67#endif
68
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -080069/*
70 * Per memcg event counter is incremented at every pagein/pageout. This counter
71 * is used for trigger some periodic events. This is straightforward and better
72 * than using jiffies etc. to handle periodic memcg event.
73 *
74 * These values will be used as !((event) & ((1 <<(thresh)) - 1))
75 */
76#define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */
77#define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080078
Balbir Singh8cdea7c2008-02-07 00:13:50 -080079/*
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080080 * Statistics for memory cgroup.
81 */
82enum mem_cgroup_stat_index {
83 /*
84 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
85 */
86 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
Balbir Singhd69b0422009-06-17 16:26:34 -070087 MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -080088 MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
Balaji Rao55e462b2008-05-01 04:35:12 -070089 MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */
90 MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */
Balbir Singh0c3e73e2009-09-23 15:56:42 -070091 MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -080092 MEM_CGROUP_EVENTS, /* incremented at every pagein/pageout */
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080093
94 MEM_CGROUP_STAT_NSTATS,
95};
96
97struct mem_cgroup_stat_cpu {
98 s64 count[MEM_CGROUP_STAT_NSTATS];
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080099};
100
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800101/*
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800102 * per-zone information in memory controller.
103 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800104struct mem_cgroup_per_zone {
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800105 /*
106 * spin_lock to protect the per cgroup LRU
107 */
Christoph Lameterb69408e2008-10-18 20:26:14 -0700108 struct list_head lists[NR_LRU_LISTS];
109 unsigned long count[NR_LRU_LISTS];
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -0800110
111 struct zone_reclaim_stat reclaim_stat;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700112 struct rb_node tree_node; /* RB tree node */
113 unsigned long long usage_in_excess;/* Set to the value by which */
114 /* the soft limit is exceeded*/
115 bool on_tree;
Balbir Singh4e416952009-09-23 15:56:39 -0700116 struct mem_cgroup *mem; /* Back pointer, we cannot */
117 /* use container_of */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800118};
119/* Macro for accessing counter */
120#define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)])
121
122struct mem_cgroup_per_node {
123 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
124};
125
126struct mem_cgroup_lru_info {
127 struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
128};
129
130/*
Balbir Singhf64c3f52009-09-23 15:56:37 -0700131 * Cgroups above their limits are maintained in a RB-Tree, independent of
132 * their hierarchy representation
133 */
134
135struct mem_cgroup_tree_per_zone {
136 struct rb_root rb_root;
137 spinlock_t lock;
138};
139
140struct mem_cgroup_tree_per_node {
141 struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
142};
143
144struct mem_cgroup_tree {
145 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
146};
147
148static struct mem_cgroup_tree soft_limit_tree __read_mostly;
149
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800150struct mem_cgroup_threshold {
151 struct eventfd_ctx *eventfd;
152 u64 threshold;
153};
154
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700155/* For threshold */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800156struct mem_cgroup_threshold_ary {
157 /* An array index points to threshold just below usage. */
Phil Carmody5407a562010-05-26 14:42:42 -0700158 int current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800159 /* Size of entries[] */
160 unsigned int size;
161 /* Array of thresholds */
162 struct mem_cgroup_threshold entries[0];
163};
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700164
165struct mem_cgroup_thresholds {
166 /* Primary thresholds array */
167 struct mem_cgroup_threshold_ary *primary;
168 /*
169 * Spare threshold array.
170 * This is needed to make mem_cgroup_unregister_event() "never fail".
171 * It must be able to store at least primary->size - 1 entries.
172 */
173 struct mem_cgroup_threshold_ary *spare;
174};
175
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700176/* for OOM */
177struct mem_cgroup_eventfd_list {
178 struct list_head list;
179 struct eventfd_ctx *eventfd;
180};
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800181
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800182static void mem_cgroup_threshold(struct mem_cgroup *mem);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700183static void mem_cgroup_oom_notify(struct mem_cgroup *mem);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800184
Balbir Singhf64c3f52009-09-23 15:56:37 -0700185/*
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800186 * The memory controller data structure. The memory controller controls both
187 * page cache and RSS per cgroup. We would eventually like to provide
188 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
189 * to help the administrator determine what knobs to tune.
190 *
191 * TODO: Add a water mark for the memory controller. Reclaim will begin when
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800192 * we hit the water mark. May be even add a low water mark, such that
193 * no reclaim occurs from a cgroup at it's low water mark, this is
194 * a feature that will be implemented much later in the future.
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800195 */
196struct mem_cgroup {
197 struct cgroup_subsys_state css;
198 /*
199 * the counter to account for memory usage
200 */
201 struct res_counter res;
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800202 /*
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800203 * the counter to account for mem+swap usage.
204 */
205 struct res_counter memsw;
206 /*
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800207 * Per cgroup active and inactive list, similar to the
208 * per zone LRU lists.
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800209 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800210 struct mem_cgroup_lru_info info;
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800211
KOSAKI Motohiro2733c062009-01-07 18:08:23 -0800212 /*
213 protect against reclaim related member.
214 */
215 spinlock_t reclaim_param_lock;
216
Balbir Singh6d61ef42009-01-07 18:08:06 -0800217 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200218 * While reclaiming in a hierarchy, we cache the last child we
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -0700219 * reclaimed from.
Balbir Singh6d61ef42009-01-07 18:08:06 -0800220 */
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -0700221 int last_scanned_child;
Balbir Singh18f59ea2009-01-07 18:08:07 -0800222 /*
223 * Should the accounting and control be hierarchical, per subtree?
224 */
225 bool use_hierarchy;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -0800226 atomic_t oom_lock;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800227 atomic_t refcnt;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800228
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -0800229 unsigned int swappiness;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -0700230 /* OOM-Killer disable */
231 int oom_kill_disable;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -0800232
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -0700233 /* set when res.limit == memsw.limit */
234 bool memsw_is_minimum;
235
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800236 /* protect arrays of thresholds */
237 struct mutex thresholds_lock;
238
239 /* thresholds for memory usage. RCU-protected */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700240 struct mem_cgroup_thresholds thresholds;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -0700241
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800242 /* thresholds for mem+swap usage. RCU-protected */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700243 struct mem_cgroup_thresholds memsw_thresholds;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -0700244
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700245 /* For oom notifier event fd */
246 struct list_head oom_notify;
247
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800248 /*
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800249 * Should we move charges of a task when a task is moved into this
250 * mem_cgroup ? And what type of charges should we move ?
251 */
252 unsigned long move_charge_at_immigrate;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800253 /*
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800254 * percpu counter.
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800255 */
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800256 struct mem_cgroup_stat_cpu *stat;
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800257};
258
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800259/* Stuffs for move charges at task migration. */
260/*
261 * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
262 * left-shifted bitmap of these types.
263 */
264enum move_type {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800265 MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700266 MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800267 NR_MOVE_TYPE,
268};
269
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800270/* "mc" and its members are protected by cgroup_mutex */
271static struct move_charge_struct {
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -0700272 spinlock_t lock; /* for from, to, moving_task */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800273 struct mem_cgroup *from;
274 struct mem_cgroup *to;
275 unsigned long precharge;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -0800276 unsigned long moved_charge;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -0800277 unsigned long moved_swap;
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800278 struct task_struct *moving_task; /* a task moving charges */
279 wait_queue_head_t waitq; /* a waitq for other context */
280} mc = {
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -0700281 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800282 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
283};
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800284
Daisuke Nishimura90254a62010-05-26 14:42:38 -0700285static bool move_anon(void)
286{
287 return test_bit(MOVE_CHARGE_TYPE_ANON,
288 &mc.to->move_charge_at_immigrate);
289}
290
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700291static bool move_file(void)
292{
293 return test_bit(MOVE_CHARGE_TYPE_FILE,
294 &mc.to->move_charge_at_immigrate);
295}
296
Balbir Singh4e416952009-09-23 15:56:39 -0700297/*
298 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
299 * limit reclaim to prevent infinite loops, if they ever occur.
300 */
301#define MEM_CGROUP_MAX_RECLAIM_LOOPS (100)
302#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2)
303
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800304enum charge_type {
305 MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
306 MEM_CGROUP_CHARGE_TYPE_MAPPED,
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700307 MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700308 MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -0800309 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -0700310 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700311 NR_CHARGE_TYPE,
312};
313
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700314/* only for here (for easy reading.) */
315#define PCGF_CACHE (1UL << PCG_CACHE)
316#define PCGF_USED (1UL << PCG_USED)
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700317#define PCGF_LOCK (1UL << PCG_LOCK)
Balbir Singh4b3bde42009-09-23 15:56:32 -0700318/* Not used, but added here for completeness */
319#define PCGF_ACCT (1UL << PCG_ACCT)
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800320
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800321/* for encoding cft->private value on file */
322#define _MEM (0)
323#define _MEMSWAP (1)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700324#define _OOM_TYPE (2)
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800325#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
326#define MEMFILE_TYPE(val) (((val) >> 16) & 0xffff)
327#define MEMFILE_ATTR(val) ((val) & 0xffff)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700328/* Used for OOM nofiier */
329#define OOM_CONTROL (0)
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800330
Balbir Singh75822b42009-09-23 15:56:38 -0700331/*
332 * Reclaim flags for mem_cgroup_hierarchical_reclaim
333 */
334#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0
335#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
336#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1
337#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
Balbir Singh4e416952009-09-23 15:56:39 -0700338#define MEM_CGROUP_RECLAIM_SOFT_BIT 0x2
339#define MEM_CGROUP_RECLAIM_SOFT (1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
Balbir Singh75822b42009-09-23 15:56:38 -0700340
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800341static void mem_cgroup_get(struct mem_cgroup *mem);
342static void mem_cgroup_put(struct mem_cgroup *mem);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -0800343static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -0800344static void drain_all_stock_async(void);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800345
Balbir Singhf64c3f52009-09-23 15:56:37 -0700346static struct mem_cgroup_per_zone *
347mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
348{
349 return &mem->info.nodeinfo[nid]->zoneinfo[zid];
350}
351
Wu Fengguangd3242362009-12-16 12:19:59 +0100352struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
353{
354 return &mem->css;
355}
356
Balbir Singhf64c3f52009-09-23 15:56:37 -0700357static struct mem_cgroup_per_zone *
358page_cgroup_zoneinfo(struct page_cgroup *pc)
359{
360 struct mem_cgroup *mem = pc->mem_cgroup;
361 int nid = page_cgroup_nid(pc);
362 int zid = page_cgroup_zid(pc);
363
364 if (!mem)
365 return NULL;
366
367 return mem_cgroup_zoneinfo(mem, nid, zid);
368}
369
370static struct mem_cgroup_tree_per_zone *
371soft_limit_tree_node_zone(int nid, int zid)
372{
373 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
374}
375
376static struct mem_cgroup_tree_per_zone *
377soft_limit_tree_from_page(struct page *page)
378{
379 int nid = page_to_nid(page);
380 int zid = page_zonenum(page);
381
382 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
383}
384
385static void
Balbir Singh4e416952009-09-23 15:56:39 -0700386__mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
Balbir Singhf64c3f52009-09-23 15:56:37 -0700387 struct mem_cgroup_per_zone *mz,
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700388 struct mem_cgroup_tree_per_zone *mctz,
389 unsigned long long new_usage_in_excess)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700390{
391 struct rb_node **p = &mctz->rb_root.rb_node;
392 struct rb_node *parent = NULL;
393 struct mem_cgroup_per_zone *mz_node;
394
395 if (mz->on_tree)
396 return;
397
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700398 mz->usage_in_excess = new_usage_in_excess;
399 if (!mz->usage_in_excess)
400 return;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700401 while (*p) {
402 parent = *p;
403 mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
404 tree_node);
405 if (mz->usage_in_excess < mz_node->usage_in_excess)
406 p = &(*p)->rb_left;
407 /*
408 * We can't avoid mem cgroups that are over their soft
409 * limit by the same amount
410 */
411 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
412 p = &(*p)->rb_right;
413 }
414 rb_link_node(&mz->tree_node, parent, p);
415 rb_insert_color(&mz->tree_node, &mctz->rb_root);
416 mz->on_tree = true;
Balbir Singh4e416952009-09-23 15:56:39 -0700417}
418
419static void
420__mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
421 struct mem_cgroup_per_zone *mz,
422 struct mem_cgroup_tree_per_zone *mctz)
423{
424 if (!mz->on_tree)
425 return;
426 rb_erase(&mz->tree_node, &mctz->rb_root);
427 mz->on_tree = false;
428}
429
430static void
Balbir Singhf64c3f52009-09-23 15:56:37 -0700431mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
432 struct mem_cgroup_per_zone *mz,
433 struct mem_cgroup_tree_per_zone *mctz)
434{
435 spin_lock(&mctz->lock);
Balbir Singh4e416952009-09-23 15:56:39 -0700436 __mem_cgroup_remove_exceeded(mem, mz, mctz);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700437 spin_unlock(&mctz->lock);
438}
439
Balbir Singhf64c3f52009-09-23 15:56:37 -0700440
441static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
442{
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700443 unsigned long long excess;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700444 struct mem_cgroup_per_zone *mz;
445 struct mem_cgroup_tree_per_zone *mctz;
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700446 int nid = page_to_nid(page);
447 int zid = page_zonenum(page);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700448 mctz = soft_limit_tree_from_page(page);
449
450 /*
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700451 * Necessary to update all ancestors when hierarchy is used.
452 * because their event counter is not touched.
Balbir Singhf64c3f52009-09-23 15:56:37 -0700453 */
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700454 for (; mem; mem = parent_mem_cgroup(mem)) {
455 mz = mem_cgroup_zoneinfo(mem, nid, zid);
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700456 excess = res_counter_soft_limit_excess(&mem->res);
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700457 /*
458 * We have to update the tree if mz is on RB-tree or
459 * mem is over its softlimit.
460 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700461 if (excess || mz->on_tree) {
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700462 spin_lock(&mctz->lock);
463 /* if on-tree, remove it */
464 if (mz->on_tree)
465 __mem_cgroup_remove_exceeded(mem, mz, mctz);
466 /*
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700467 * Insert again. mz->usage_in_excess will be updated.
468 * If excess is 0, no tree ops.
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700469 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700470 __mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700471 spin_unlock(&mctz->lock);
472 }
Balbir Singhf64c3f52009-09-23 15:56:37 -0700473 }
474}
475
476static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
477{
478 int node, zone;
479 struct mem_cgroup_per_zone *mz;
480 struct mem_cgroup_tree_per_zone *mctz;
481
482 for_each_node_state(node, N_POSSIBLE) {
483 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
484 mz = mem_cgroup_zoneinfo(mem, node, zone);
485 mctz = soft_limit_tree_node_zone(node, zone);
486 mem_cgroup_remove_exceeded(mem, mz, mctz);
487 }
488 }
489}
490
Balbir Singh4e416952009-09-23 15:56:39 -0700491static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem)
492{
493 return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT;
494}
495
496static struct mem_cgroup_per_zone *
497__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
498{
499 struct rb_node *rightmost = NULL;
KAMEZAWA Hiroyuki26251ea2009-10-01 15:44:08 -0700500 struct mem_cgroup_per_zone *mz;
Balbir Singh4e416952009-09-23 15:56:39 -0700501
502retry:
KAMEZAWA Hiroyuki26251ea2009-10-01 15:44:08 -0700503 mz = NULL;
Balbir Singh4e416952009-09-23 15:56:39 -0700504 rightmost = rb_last(&mctz->rb_root);
505 if (!rightmost)
506 goto done; /* Nothing to reclaim from */
507
508 mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
509 /*
510 * Remove the node now but someone else can add it back,
511 * we will to add it back at the end of reclaim to its correct
512 * position in the tree.
513 */
514 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
515 if (!res_counter_soft_limit_excess(&mz->mem->res) ||
516 !css_tryget(&mz->mem->css))
517 goto retry;
518done:
519 return mz;
520}
521
522static struct mem_cgroup_per_zone *
523mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
524{
525 struct mem_cgroup_per_zone *mz;
526
527 spin_lock(&mctz->lock);
528 mz = __mem_cgroup_largest_soft_limit_node(mctz);
529 spin_unlock(&mctz->lock);
530 return mz;
531}
532
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800533static s64 mem_cgroup_read_stat(struct mem_cgroup *mem,
534 enum mem_cgroup_stat_index idx)
535{
536 int cpu;
537 s64 val = 0;
538
539 for_each_possible_cpu(cpu)
540 val += per_cpu(mem->stat->count[idx], cpu);
541 return val;
542}
543
544static s64 mem_cgroup_local_usage(struct mem_cgroup *mem)
545{
546 s64 ret;
547
548 ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
549 ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
550 return ret;
551}
552
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700553static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
554 bool charge)
555{
556 int val = (charge) ? 1 : -1;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800557 this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700558}
559
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700560static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
561 struct page_cgroup *pc,
562 bool charge)
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800563{
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700564 int val = (charge) ? 1 : -1;
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800565
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800566 preempt_disable();
567
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700568 if (PageCgroupCache(pc))
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800569 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], val);
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800570 else
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800571 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], val);
Balaji Rao55e462b2008-05-01 04:35:12 -0700572
573 if (charge)
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800574 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]);
Balaji Rao55e462b2008-05-01 04:35:12 -0700575 else
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800576 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]);
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800577 __this_cpu_inc(mem->stat->count[MEM_CGROUP_EVENTS]);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800578
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800579 preempt_enable();
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800580}
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800581
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700582static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
Christoph Lameterb69408e2008-10-18 20:26:14 -0700583 enum lru_list idx)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800584{
585 int nid, zid;
586 struct mem_cgroup_per_zone *mz;
587 u64 total = 0;
588
589 for_each_online_node(nid)
590 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
591 mz = mem_cgroup_zoneinfo(mem, nid, zid);
592 total += MEM_CGROUP_ZSTAT(mz, idx);
593 }
594 return total;
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800595}
596
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800597static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift)
598{
599 s64 val;
600
601 val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]);
602
603 return !(val & ((1 << event_mask_shift) - 1));
604}
605
606/*
607 * Check events in order.
608 *
609 */
610static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
611{
612 /* threshold event is triggered in finer grain than soft limit */
613 if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) {
614 mem_cgroup_threshold(mem);
615 if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH)))
616 mem_cgroup_update_tree(mem, page);
617 }
618}
619
Hugh Dickinsd5b69e32008-03-04 14:29:10 -0800620static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800621{
622 return container_of(cgroup_subsys_state(cont,
623 mem_cgroup_subsys_id), struct mem_cgroup,
624 css);
625}
626
Balbir Singhcf475ad2008-04-29 01:00:16 -0700627struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800628{
Balbir Singh31a78f22008-09-28 23:09:31 +0100629 /*
630 * mm_update_next_owner() may clear mm->owner to NULL
631 * if it races with swapoff, page migration, etc.
632 * So this can be called with p == NULL.
633 */
634 if (unlikely(!p))
635 return NULL;
636
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800637 return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
638 struct mem_cgroup, css);
639}
640
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800641static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
642{
643 struct mem_cgroup *mem = NULL;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700644
645 if (!mm)
646 return NULL;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800647 /*
648 * Because we have no locks, mm->owner's may be being moved to other
649 * cgroup. We use css_tryget() here even if this looks
650 * pessimistic (rather than adding locks here).
651 */
652 rcu_read_lock();
653 do {
654 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
655 if (unlikely(!mem))
656 break;
657 } while (!css_tryget(&mem->css));
658 rcu_read_unlock();
659 return mem;
660}
661
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700662/*
663 * Call callback function against all cgroup under hierarchy tree.
664 */
665static int mem_cgroup_walk_tree(struct mem_cgroup *root, void *data,
666 int (*func)(struct mem_cgroup *, void *))
667{
668 int found, ret, nextid;
669 struct cgroup_subsys_state *css;
670 struct mem_cgroup *mem;
671
672 if (!root->use_hierarchy)
673 return (*func)(root, data);
674
675 nextid = 1;
676 do {
677 ret = 0;
678 mem = NULL;
679
680 rcu_read_lock();
681 css = css_get_next(&mem_cgroup_subsys, nextid, &root->css,
682 &found);
683 if (css && css_tryget(css))
684 mem = container_of(css, struct mem_cgroup, css);
685 rcu_read_unlock();
686
687 if (mem) {
688 ret = (*func)(mem, data);
689 css_put(&mem->css);
690 }
691 nextid = found + 1;
692 } while (!ret && css);
693
694 return ret;
695}
696
Balbir Singh4b3bde42009-09-23 15:56:32 -0700697static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
698{
699 return (mem == root_mem_cgroup);
700}
701
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800702/*
703 * Following LRU functions are allowed to be used without PCG_LOCK.
704 * Operations are called by routine of global LRU independently from memcg.
705 * What we have to take care of here is validness of pc->mem_cgroup.
706 *
707 * Changes to pc->mem_cgroup happens when
708 * 1. charge
709 * 2. moving account
710 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
711 * It is added to LRU before charge.
712 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
713 * When moving account, the page is not on LRU. It's isolated.
714 */
715
716void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800717{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800718 struct page_cgroup *pc;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800719 struct mem_cgroup_per_zone *mz;
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700720
Hirokazu Takahashif8d66542009-01-07 18:08:02 -0800721 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800722 return;
723 pc = lookup_page_cgroup(page);
724 /* can happen while we handle swapcache. */
Balbir Singh4b3bde42009-09-23 15:56:32 -0700725 if (!TestClearPageCgroupAcctLRU(pc))
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800726 return;
Balbir Singh4b3bde42009-09-23 15:56:32 -0700727 VM_BUG_ON(!pc->mem_cgroup);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800728 /*
729 * We don't check PCG_USED bit. It's cleared when the "page" is finally
730 * removed from global LRU.
731 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800732 mz = page_cgroup_zoneinfo(pc);
Christoph Lameterb69408e2008-10-18 20:26:14 -0700733 MEM_CGROUP_ZSTAT(mz, lru) -= 1;
Balbir Singh4b3bde42009-09-23 15:56:32 -0700734 if (mem_cgroup_is_root(pc->mem_cgroup))
735 return;
736 VM_BUG_ON(list_empty(&pc->lru));
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800737 list_del_init(&pc->lru);
738 return;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800739}
740
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800741void mem_cgroup_del_lru(struct page *page)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800742{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800743 mem_cgroup_del_lru_list(page, page_lru(page));
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800744}
745
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800746void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
Balbir Singh66e17072008-02-07 00:13:56 -0800747{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800748 struct mem_cgroup_per_zone *mz;
749 struct page_cgroup *pc;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800750
Hirokazu Takahashif8d66542009-01-07 18:08:02 -0800751 if (mem_cgroup_disabled())
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700752 return;
Christoph Lameterb69408e2008-10-18 20:26:14 -0700753
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800754 pc = lookup_page_cgroup(page);
Daisuke Nishimurabd112db2009-01-15 13:51:11 -0800755 /*
756 * Used bit is set without atomic ops but after smp_wmb().
757 * For making pc->mem_cgroup visible, insert smp_rmb() here.
758 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800759 smp_rmb();
Balbir Singh4b3bde42009-09-23 15:56:32 -0700760 /* unused or root page is not rotated. */
761 if (!PageCgroupUsed(pc) || mem_cgroup_is_root(pc->mem_cgroup))
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800762 return;
763 mz = page_cgroup_zoneinfo(pc);
Christoph Lameterb69408e2008-10-18 20:26:14 -0700764 list_move(&pc->lru, &mz->lists[lru]);
Balbir Singh66e17072008-02-07 00:13:56 -0800765}
766
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800767void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
768{
769 struct page_cgroup *pc;
770 struct mem_cgroup_per_zone *mz;
771
Hirokazu Takahashif8d66542009-01-07 18:08:02 -0800772 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800773 return;
774 pc = lookup_page_cgroup(page);
Balbir Singh4b3bde42009-09-23 15:56:32 -0700775 VM_BUG_ON(PageCgroupAcctLRU(pc));
Daisuke Nishimurabd112db2009-01-15 13:51:11 -0800776 /*
777 * Used bit is set without atomic ops but after smp_wmb().
778 * For making pc->mem_cgroup visible, insert smp_rmb() here.
779 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800780 smp_rmb();
781 if (!PageCgroupUsed(pc))
782 return;
783
784 mz = page_cgroup_zoneinfo(pc);
785 MEM_CGROUP_ZSTAT(mz, lru) += 1;
Balbir Singh4b3bde42009-09-23 15:56:32 -0700786 SetPageCgroupAcctLRU(pc);
787 if (mem_cgroup_is_root(pc->mem_cgroup))
788 return;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800789 list_add(&pc->lru, &mz->lists[lru]);
790}
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800791
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800792/*
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800793 * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to
794 * lru because the page may.be reused after it's fully uncharged (because of
795 * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge
796 * it again. This function is only used to charge SwapCache. It's done under
797 * lock_page and expected that zone->lru_lock is never held.
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800798 */
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800799static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page)
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800800{
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800801 unsigned long flags;
802 struct zone *zone = page_zone(page);
803 struct page_cgroup *pc = lookup_page_cgroup(page);
804
805 spin_lock_irqsave(&zone->lru_lock, flags);
806 /*
807 * Forget old LRU when this page_cgroup is *not* used. This Used bit
808 * is guarded by lock_page() because the page is SwapCache.
809 */
810 if (!PageCgroupUsed(pc))
811 mem_cgroup_del_lru_list(page, page_lru(page));
812 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800813}
814
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800815static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
816{
817 unsigned long flags;
818 struct zone *zone = page_zone(page);
819 struct page_cgroup *pc = lookup_page_cgroup(page);
820
821 spin_lock_irqsave(&zone->lru_lock, flags);
822 /* link when the page is linked to LRU but page_cgroup isn't */
Balbir Singh4b3bde42009-09-23 15:56:32 -0700823 if (PageLRU(page) && !PageCgroupAcctLRU(pc))
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800824 mem_cgroup_add_lru_list(page, page_lru(page));
825 spin_unlock_irqrestore(&zone->lru_lock, flags);
826}
827
828
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800829void mem_cgroup_move_lists(struct page *page,
830 enum lru_list from, enum lru_list to)
831{
Hirokazu Takahashif8d66542009-01-07 18:08:02 -0800832 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800833 return;
834 mem_cgroup_del_lru_list(page, from);
835 mem_cgroup_add_lru_list(page, to);
836}
837
David Rientjes4c4a2212008-02-07 00:14:06 -0800838int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
839{
840 int ret;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700841 struct mem_cgroup *curr = NULL;
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -0700842 struct task_struct *p;
David Rientjes4c4a2212008-02-07 00:14:06 -0800843
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -0700844 p = find_lock_task_mm(task);
845 if (!p)
846 return 0;
847 curr = try_get_mem_cgroup_from_mm(p->mm);
848 task_unlock(p);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700849 if (!curr)
850 return 0;
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -0800851 /*
852 * We should check use_hierarchy of "mem" not "curr". Because checking
853 * use_hierarchy of "curr" here make this function true if hierarchy is
854 * enabled in "curr" and "curr" is a child of "mem" in *cgroup*
855 * hierarchy(even if use_hierarchy is disabled in "mem").
856 */
857 if (mem->use_hierarchy)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700858 ret = css_is_ancestor(&curr->css, &mem->css);
859 else
860 ret = (curr == mem);
861 css_put(&curr->css);
David Rientjes4c4a2212008-02-07 00:14:06 -0800862 return ret;
863}
864
KOSAKI Motohiroc772be92009-01-07 18:08:25 -0800865static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800866{
867 unsigned long active;
868 unsigned long inactive;
KOSAKI Motohiroc772be92009-01-07 18:08:25 -0800869 unsigned long gb;
870 unsigned long inactive_ratio;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800871
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700872 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
873 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800874
KOSAKI Motohiroc772be92009-01-07 18:08:25 -0800875 gb = (inactive + active) >> (30 - PAGE_SHIFT);
876 if (gb)
877 inactive_ratio = int_sqrt(10 * gb);
878 else
879 inactive_ratio = 1;
880
881 if (present_pages) {
882 present_pages[0] = inactive;
883 present_pages[1] = active;
884 }
885
886 return inactive_ratio;
887}
888
889int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
890{
891 unsigned long active;
892 unsigned long inactive;
893 unsigned long present_pages[2];
894 unsigned long inactive_ratio;
895
896 inactive_ratio = calc_inactive_ratio(memcg, present_pages);
897
898 inactive = present_pages[0];
899 active = present_pages[1];
900
901 if (inactive * inactive_ratio < active)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800902 return 1;
903
904 return 0;
905}
906
Rik van Riel56e49d22009-06-16 15:32:28 -0700907int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
908{
909 unsigned long active;
910 unsigned long inactive;
911
912 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
913 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
914
915 return (active > inactive);
916}
917
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -0800918unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
919 struct zone *zone,
920 enum lru_list lru)
921{
KOSAKI Motohiro13d7e3a2010-08-10 18:03:06 -0700922 int nid = zone_to_nid(zone);
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -0800923 int zid = zone_idx(zone);
924 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
925
926 return MEM_CGROUP_ZSTAT(mz, lru);
927}
928
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -0800929struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
930 struct zone *zone)
931{
KOSAKI Motohiro13d7e3a2010-08-10 18:03:06 -0700932 int nid = zone_to_nid(zone);
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -0800933 int zid = zone_idx(zone);
934 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
935
936 return &mz->reclaim_stat;
937}
938
939struct zone_reclaim_stat *
940mem_cgroup_get_reclaim_stat_from_page(struct page *page)
941{
942 struct page_cgroup *pc;
943 struct mem_cgroup_per_zone *mz;
944
945 if (mem_cgroup_disabled())
946 return NULL;
947
948 pc = lookup_page_cgroup(page);
Daisuke Nishimurabd112db2009-01-15 13:51:11 -0800949 /*
950 * Used bit is set without atomic ops but after smp_wmb().
951 * For making pc->mem_cgroup visible, insert smp_rmb() here.
952 */
953 smp_rmb();
954 if (!PageCgroupUsed(pc))
955 return NULL;
956
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -0800957 mz = page_cgroup_zoneinfo(pc);
958 if (!mz)
959 return NULL;
960
961 return &mz->reclaim_stat;
962}
963
Balbir Singh66e17072008-02-07 00:13:56 -0800964unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
965 struct list_head *dst,
966 unsigned long *scanned, int order,
967 int mode, struct zone *z,
968 struct mem_cgroup *mem_cont,
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700969 int active, int file)
Balbir Singh66e17072008-02-07 00:13:56 -0800970{
971 unsigned long nr_taken = 0;
972 struct page *page;
973 unsigned long scan;
974 LIST_HEAD(pc_list);
975 struct list_head *src;
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -0800976 struct page_cgroup *pc, *tmp;
KOSAKI Motohiro13d7e3a2010-08-10 18:03:06 -0700977 int nid = zone_to_nid(z);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -0800978 int zid = zone_idx(z);
979 struct mem_cgroup_per_zone *mz;
Johannes Weinerb7c46d12009-09-21 17:02:56 -0700980 int lru = LRU_FILE * file + active;
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -0700981 int ret;
Balbir Singh66e17072008-02-07 00:13:56 -0800982
Balbir Singhcf475ad2008-04-29 01:00:16 -0700983 BUG_ON(!mem_cont);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -0800984 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
Christoph Lameterb69408e2008-10-18 20:26:14 -0700985 src = &mz->lists[lru];
Balbir Singh66e17072008-02-07 00:13:56 -0800986
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -0800987 scan = 0;
988 list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
Hugh Dickins436c65412008-02-07 00:14:12 -0800989 if (scan >= nr_to_scan)
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -0800990 break;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800991
992 page = pc->page;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700993 if (unlikely(!PageCgroupUsed(pc)))
994 continue;
Hugh Dickins436c65412008-02-07 00:14:12 -0800995 if (unlikely(!PageLRU(page)))
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -0800996 continue;
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -0800997
Hugh Dickins436c65412008-02-07 00:14:12 -0800998 scan++;
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -0700999 ret = __isolate_lru_page(page, mode, file);
1000 switch (ret) {
1001 case 0:
Balbir Singh66e17072008-02-07 00:13:56 -08001002 list_move(&page->lru, dst);
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001003 mem_cgroup_del_lru(page);
Balbir Singh66e17072008-02-07 00:13:56 -08001004 nr_taken++;
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001005 break;
1006 case -EBUSY:
1007 /* we don't affect global LRU but rotate in our LRU */
1008 mem_cgroup_rotate_lru_list(page, page_lru(page));
1009 break;
1010 default:
1011 break;
Balbir Singh66e17072008-02-07 00:13:56 -08001012 }
1013 }
1014
Balbir Singh66e17072008-02-07 00:13:56 -08001015 *scanned = scan;
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -07001016
1017 trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken,
1018 0, 0, 0, mode);
1019
Balbir Singh66e17072008-02-07 00:13:56 -08001020 return nr_taken;
1021}
1022
Balbir Singh6d61ef42009-01-07 18:08:06 -08001023#define mem_cgroup_from_res_counter(counter, member) \
1024 container_of(counter, struct mem_cgroup, member)
1025
Daisuke Nishimurab85a96c2009-01-07 18:08:12 -08001026static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
1027{
1028 if (do_swap_account) {
1029 if (res_counter_check_under_limit(&mem->res) &&
1030 res_counter_check_under_limit(&mem->memsw))
1031 return true;
1032 } else
1033 if (res_counter_check_under_limit(&mem->res))
1034 return true;
1035 return false;
1036}
1037
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001038static unsigned int get_swappiness(struct mem_cgroup *memcg)
1039{
1040 struct cgroup *cgrp = memcg->css.cgroup;
1041 unsigned int swappiness;
1042
1043 /* root ? */
1044 if (cgrp->parent == NULL)
1045 return vm_swappiness;
1046
1047 spin_lock(&memcg->reclaim_param_lock);
1048 swappiness = memcg->swappiness;
1049 spin_unlock(&memcg->reclaim_param_lock);
1050
1051 return swappiness;
1052}
1053
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001054/* A routine for testing mem is not under move_account */
1055
1056static bool mem_cgroup_under_move(struct mem_cgroup *mem)
1057{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001058 struct mem_cgroup *from;
1059 struct mem_cgroup *to;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001060 bool ret = false;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001061 /*
1062 * Unlike task_move routines, we access mc.to, mc.from not under
1063 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1064 */
1065 spin_lock(&mc.lock);
1066 from = mc.from;
1067 to = mc.to;
1068 if (!from)
1069 goto unlock;
1070 if (from == mem || to == mem
1071 || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css))
1072 || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css)))
1073 ret = true;
1074unlock:
1075 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001076 return ret;
1077}
1078
1079static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem)
1080{
1081 if (mc.moving_task && current != mc.moving_task) {
1082 if (mem_cgroup_under_move(mem)) {
1083 DEFINE_WAIT(wait);
1084 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1085 /* moving charge context might have finished. */
1086 if (mc.moving_task)
1087 schedule();
1088 finish_wait(&mc.waitq, &wait);
1089 return true;
1090 }
1091 }
1092 return false;
1093}
1094
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001095static int mem_cgroup_count_children_cb(struct mem_cgroup *mem, void *data)
1096{
1097 int *val = data;
1098 (*val)++;
1099 return 0;
1100}
Balbir Singhe2224322009-04-02 16:57:39 -07001101
1102/**
Kirill A. Shutemov6a6135b2010-03-10 15:22:25 -08001103 * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
Balbir Singhe2224322009-04-02 16:57:39 -07001104 * @memcg: The memory cgroup that went over limit
1105 * @p: Task that is going to be killed
1106 *
1107 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1108 * enabled
1109 */
1110void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1111{
1112 struct cgroup *task_cgrp;
1113 struct cgroup *mem_cgrp;
1114 /*
1115 * Need a buffer in BSS, can't rely on allocations. The code relies
1116 * on the assumption that OOM is serialized for memory controller.
1117 * If this assumption is broken, revisit this code.
1118 */
1119 static char memcg_name[PATH_MAX];
1120 int ret;
1121
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -08001122 if (!memcg || !p)
Balbir Singhe2224322009-04-02 16:57:39 -07001123 return;
1124
1125
1126 rcu_read_lock();
1127
1128 mem_cgrp = memcg->css.cgroup;
1129 task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1130
1131 ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1132 if (ret < 0) {
1133 /*
1134 * Unfortunately, we are unable to convert to a useful name
1135 * But we'll still print out the usage information
1136 */
1137 rcu_read_unlock();
1138 goto done;
1139 }
1140 rcu_read_unlock();
1141
1142 printk(KERN_INFO "Task in %s killed", memcg_name);
1143
1144 rcu_read_lock();
1145 ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1146 if (ret < 0) {
1147 rcu_read_unlock();
1148 goto done;
1149 }
1150 rcu_read_unlock();
1151
1152 /*
1153 * Continues from above, so we don't need an KERN_ level
1154 */
1155 printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
1156done:
1157
1158 printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
1159 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1160 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1161 res_counter_read_u64(&memcg->res, RES_FAILCNT));
1162 printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
1163 "failcnt %llu\n",
1164 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1165 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1166 res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1167}
1168
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001169/*
1170 * This function returns the number of memcg under hierarchy tree. Returns
1171 * 1(self count) if no children.
1172 */
1173static int mem_cgroup_count_children(struct mem_cgroup *mem)
1174{
1175 int num = 0;
1176 mem_cgroup_walk_tree(mem, &num, mem_cgroup_count_children_cb);
1177 return num;
1178}
1179
Balbir Singh6d61ef42009-01-07 18:08:06 -08001180/*
David Rientjesa63d83f2010-08-09 17:19:46 -07001181 * Return the memory (and swap, if configured) limit for a memcg.
1182 */
1183u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
1184{
1185 u64 limit;
1186 u64 memsw;
1187
1188 limit = res_counter_read_u64(&memcg->res, RES_LIMIT) +
1189 total_swap_pages;
1190 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1191 /*
1192 * If memsw is finite and limits the amount of swap space available
1193 * to this memcg, return that limit.
1194 */
1195 return min(limit, memsw);
1196}
1197
1198/*
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001199 * Visit the first child (need not be the first child as per the ordering
1200 * of the cgroup list, since we track last_scanned_child) of @mem and use
1201 * that to reclaim free pages from.
1202 */
1203static struct mem_cgroup *
1204mem_cgroup_select_victim(struct mem_cgroup *root_mem)
1205{
1206 struct mem_cgroup *ret = NULL;
1207 struct cgroup_subsys_state *css;
1208 int nextid, found;
1209
1210 if (!root_mem->use_hierarchy) {
1211 css_get(&root_mem->css);
1212 ret = root_mem;
1213 }
1214
1215 while (!ret) {
1216 rcu_read_lock();
1217 nextid = root_mem->last_scanned_child + 1;
1218 css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
1219 &found);
1220 if (css && css_tryget(css))
1221 ret = container_of(css, struct mem_cgroup, css);
1222
1223 rcu_read_unlock();
1224 /* Updates scanning parameter */
1225 spin_lock(&root_mem->reclaim_param_lock);
1226 if (!css) {
1227 /* this means start scan from ID:1 */
1228 root_mem->last_scanned_child = 0;
1229 } else
1230 root_mem->last_scanned_child = found;
1231 spin_unlock(&root_mem->reclaim_param_lock);
1232 }
1233
1234 return ret;
1235}
1236
1237/*
1238 * Scan the hierarchy if needed to reclaim memory. We remember the last child
1239 * we reclaimed from, so that we don't end up penalizing one child extensively
1240 * based on its position in the children list.
Balbir Singh6d61ef42009-01-07 18:08:06 -08001241 *
1242 * root_mem is the original ancestor that we've been reclaim from.
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001243 *
1244 * We give up and return to the caller when we visit root_mem twice.
1245 * (other groups can be removed while we're walking....)
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001246 *
1247 * If shrink==true, for avoiding to free too much, this returns immedieately.
Balbir Singh6d61ef42009-01-07 18:08:06 -08001248 */
1249static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
Balbir Singh4e416952009-09-23 15:56:39 -07001250 struct zone *zone,
Balbir Singh75822b42009-09-23 15:56:38 -07001251 gfp_t gfp_mask,
1252 unsigned long reclaim_options)
Balbir Singh6d61ef42009-01-07 18:08:06 -08001253{
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001254 struct mem_cgroup *victim;
1255 int ret, total = 0;
1256 int loop = 0;
Balbir Singh75822b42009-09-23 15:56:38 -07001257 bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
1258 bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
Balbir Singh4e416952009-09-23 15:56:39 -07001259 bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
1260 unsigned long excess = mem_cgroup_get_excess(root_mem);
Balbir Singh6d61ef42009-01-07 18:08:06 -08001261
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07001262 /* If memsw_is_minimum==1, swap-out is of-no-use. */
1263 if (root_mem->memsw_is_minimum)
1264 noswap = true;
1265
Balbir Singh4e416952009-09-23 15:56:39 -07001266 while (1) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001267 victim = mem_cgroup_select_victim(root_mem);
Balbir Singh4e416952009-09-23 15:56:39 -07001268 if (victim == root_mem) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001269 loop++;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001270 if (loop >= 1)
1271 drain_all_stock_async();
Balbir Singh4e416952009-09-23 15:56:39 -07001272 if (loop >= 2) {
1273 /*
1274 * If we have not been able to reclaim
1275 * anything, it might because there are
1276 * no reclaimable pages under this hierarchy
1277 */
1278 if (!check_soft || !total) {
1279 css_put(&victim->css);
1280 break;
1281 }
1282 /*
1283 * We want to do more targetted reclaim.
1284 * excess >> 2 is not to excessive so as to
1285 * reclaim too much, nor too less that we keep
1286 * coming back to reclaim from this cgroup
1287 */
1288 if (total >= (excess >> 2) ||
1289 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
1290 css_put(&victim->css);
1291 break;
1292 }
1293 }
1294 }
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08001295 if (!mem_cgroup_local_usage(victim)) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001296 /* this cgroup's local usage == 0 */
1297 css_put(&victim->css);
Balbir Singh6d61ef42009-01-07 18:08:06 -08001298 continue;
1299 }
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001300 /* we use swappiness of local cgroup */
Balbir Singh4e416952009-09-23 15:56:39 -07001301 if (check_soft)
1302 ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
KOSAKI Motohiro14fec792010-08-10 18:03:05 -07001303 noswap, get_swappiness(victim), zone);
Balbir Singh4e416952009-09-23 15:56:39 -07001304 else
1305 ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
1306 noswap, get_swappiness(victim));
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001307 css_put(&victim->css);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001308 /*
1309 * At shrinking usage, we can't check we should stop here or
1310 * reclaim more. It's depends on callers. last_scanned_child
1311 * will work enough for keeping fairness under tree.
1312 */
1313 if (shrink)
1314 return ret;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001315 total += ret;
Balbir Singh4e416952009-09-23 15:56:39 -07001316 if (check_soft) {
1317 if (res_counter_check_under_soft_limit(&root_mem->res))
1318 return total;
1319 } else if (mem_cgroup_check_under_limit(root_mem))
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001320 return 1 + total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001321 }
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001322 return total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001323}
1324
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001325static int mem_cgroup_oom_lock_cb(struct mem_cgroup *mem, void *data)
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001326{
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001327 int *val = (int *)data;
1328 int x;
1329 /*
1330 * Logically, we can stop scanning immediately when we find
1331 * a memcg is already locked. But condidering unlock ops and
1332 * creation/removal of memcg, scan-all is simple operation.
1333 */
1334 x = atomic_inc_return(&mem->oom_lock);
1335 *val = max(x, *val);
1336 return 0;
1337}
1338/*
1339 * Check OOM-Killer is already running under our hierarchy.
1340 * If someone is running, return false.
1341 */
1342static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
1343{
1344 int lock_count = 0;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001345
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001346 mem_cgroup_walk_tree(mem, &lock_count, mem_cgroup_oom_lock_cb);
1347
1348 if (lock_count == 1)
1349 return true;
1350 return false;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001351}
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001352
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001353static int mem_cgroup_oom_unlock_cb(struct mem_cgroup *mem, void *data)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001354{
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001355 /*
1356 * When a new child is created while the hierarchy is under oom,
1357 * mem_cgroup_oom_lock() may not be called. We have to use
1358 * atomic_add_unless() here.
1359 */
1360 atomic_add_unless(&mem->oom_lock, -1, 0);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001361 return 0;
1362}
1363
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001364static void mem_cgroup_oom_unlock(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001365{
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001366 mem_cgroup_walk_tree(mem, NULL, mem_cgroup_oom_unlock_cb);
1367}
1368
1369static DEFINE_MUTEX(memcg_oom_mutex);
1370static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1371
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001372struct oom_wait_info {
1373 struct mem_cgroup *mem;
1374 wait_queue_t wait;
1375};
1376
1377static int memcg_oom_wake_function(wait_queue_t *wait,
1378 unsigned mode, int sync, void *arg)
1379{
1380 struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg;
1381 struct oom_wait_info *oom_wait_info;
1382
1383 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1384
1385 if (oom_wait_info->mem == wake_mem)
1386 goto wakeup;
1387 /* if no hierarchy, no match */
1388 if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy)
1389 return 0;
1390 /*
1391 * Both of oom_wait_info->mem and wake_mem are stable under us.
1392 * Then we can use css_is_ancestor without taking care of RCU.
1393 */
1394 if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) &&
1395 !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css))
1396 return 0;
1397
1398wakeup:
1399 return autoremove_wake_function(wait, mode, sync, arg);
1400}
1401
1402static void memcg_wakeup_oom(struct mem_cgroup *mem)
1403{
1404 /* for filtering, pass "mem" as argument. */
1405 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem);
1406}
1407
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001408static void memcg_oom_recover(struct mem_cgroup *mem)
1409{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001410 if (mem && atomic_read(&mem->oom_lock))
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001411 memcg_wakeup_oom(mem);
1412}
1413
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001414/*
1415 * try to call OOM killer. returns false if we should exit memory-reclaim loop.
1416 */
1417bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
1418{
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001419 struct oom_wait_info owait;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001420 bool locked, need_to_kill;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001421
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001422 owait.mem = mem;
1423 owait.wait.flags = 0;
1424 owait.wait.func = memcg_oom_wake_function;
1425 owait.wait.private = current;
1426 INIT_LIST_HEAD(&owait.wait.task_list);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001427 need_to_kill = true;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001428 /* At first, try to OOM lock hierarchy under mem.*/
1429 mutex_lock(&memcg_oom_mutex);
1430 locked = mem_cgroup_oom_lock(mem);
1431 /*
1432 * Even if signal_pending(), we can't quit charge() loop without
1433 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
1434 * under OOM is always welcomed, use TASK_KILLABLE here.
1435 */
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001436 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1437 if (!locked || mem->oom_kill_disable)
1438 need_to_kill = false;
1439 if (locked)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07001440 mem_cgroup_oom_notify(mem);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001441 mutex_unlock(&memcg_oom_mutex);
1442
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001443 if (need_to_kill) {
1444 finish_wait(&memcg_oom_waitq, &owait.wait);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001445 mem_cgroup_out_of_memory(mem, mask);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001446 } else {
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001447 schedule();
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001448 finish_wait(&memcg_oom_waitq, &owait.wait);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001449 }
1450 mutex_lock(&memcg_oom_mutex);
1451 mem_cgroup_oom_unlock(mem);
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001452 memcg_wakeup_oom(mem);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001453 mutex_unlock(&memcg_oom_mutex);
1454
1455 if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
1456 return false;
1457 /* Give chance to dying process */
1458 schedule_timeout(1);
1459 return true;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001460}
1461
Balbir Singhd69b0422009-06-17 16:26:34 -07001462/*
1463 * Currently used to update mapped file statistics, but the routine can be
1464 * generalized to update other statistics as well.
1465 */
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -08001466void mem_cgroup_update_file_mapped(struct page *page, int val)
Balbir Singhd69b0422009-06-17 16:26:34 -07001467{
1468 struct mem_cgroup *mem;
Balbir Singhd69b0422009-06-17 16:26:34 -07001469 struct page_cgroup *pc;
1470
Balbir Singhd69b0422009-06-17 16:26:34 -07001471 pc = lookup_page_cgroup(page);
1472 if (unlikely(!pc))
1473 return;
1474
1475 lock_page_cgroup(pc);
1476 mem = pc->mem_cgroup;
KAMEZAWA Hiroyuki8725d542010-04-06 14:35:05 -07001477 if (!mem || !PageCgroupUsed(pc))
Balbir Singhd69b0422009-06-17 16:26:34 -07001478 goto done;
1479
1480 /*
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08001481 * Preemption is already disabled. We can use __this_cpu_xxx
Balbir Singhd69b0422009-06-17 16:26:34 -07001482 */
KAMEZAWA Hiroyuki8725d542010-04-06 14:35:05 -07001483 if (val > 0) {
1484 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
1485 SetPageCgroupFileMapped(pc);
1486 } else {
1487 __this_cpu_dec(mem->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
1488 ClearPageCgroupFileMapped(pc);
1489 }
Balbir Singhd69b0422009-06-17 16:26:34 -07001490
Balbir Singhd69b0422009-06-17 16:26:34 -07001491done:
1492 unlock_page_cgroup(pc);
1493}
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001494
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001495/*
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001496 * size of first charge trial. "32" comes from vmscan.c's magic value.
1497 * TODO: maybe necessary to use big numbers in big irons.
1498 */
1499#define CHARGE_SIZE (32 * PAGE_SIZE)
1500struct memcg_stock_pcp {
1501 struct mem_cgroup *cached; /* this never be root cgroup */
1502 int charge;
1503 struct work_struct work;
1504};
1505static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1506static atomic_t memcg_drain_count;
1507
1508/*
1509 * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed
1510 * from local stock and true is returned. If the stock is 0 or charges from a
1511 * cgroup which is not current target, returns false. This stock will be
1512 * refilled.
1513 */
1514static bool consume_stock(struct mem_cgroup *mem)
1515{
1516 struct memcg_stock_pcp *stock;
1517 bool ret = true;
1518
1519 stock = &get_cpu_var(memcg_stock);
1520 if (mem == stock->cached && stock->charge)
1521 stock->charge -= PAGE_SIZE;
1522 else /* need to call res_counter_charge */
1523 ret = false;
1524 put_cpu_var(memcg_stock);
1525 return ret;
1526}
1527
1528/*
1529 * Returns stocks cached in percpu to res_counter and reset cached information.
1530 */
1531static void drain_stock(struct memcg_stock_pcp *stock)
1532{
1533 struct mem_cgroup *old = stock->cached;
1534
1535 if (stock->charge) {
1536 res_counter_uncharge(&old->res, stock->charge);
1537 if (do_swap_account)
1538 res_counter_uncharge(&old->memsw, stock->charge);
1539 }
1540 stock->cached = NULL;
1541 stock->charge = 0;
1542}
1543
1544/*
1545 * This must be called under preempt disabled or must be called by
1546 * a thread which is pinned to local cpu.
1547 */
1548static void drain_local_stock(struct work_struct *dummy)
1549{
1550 struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
1551 drain_stock(stock);
1552}
1553
1554/*
1555 * Cache charges(val) which is from res_counter, to local per_cpu area.
Greg Thelen320cc512010-03-15 15:27:28 +01001556 * This will be consumed by consume_stock() function, later.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001557 */
1558static void refill_stock(struct mem_cgroup *mem, int val)
1559{
1560 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
1561
1562 if (stock->cached != mem) { /* reset if necessary */
1563 drain_stock(stock);
1564 stock->cached = mem;
1565 }
1566 stock->charge += val;
1567 put_cpu_var(memcg_stock);
1568}
1569
1570/*
1571 * Tries to drain stocked charges in other cpus. This function is asynchronous
1572 * and just put a work per cpu for draining localy on each cpu. Caller can
1573 * expects some charges will be back to res_counter later but cannot wait for
1574 * it.
1575 */
1576static void drain_all_stock_async(void)
1577{
1578 int cpu;
1579 /* This function is for scheduling "drain" in asynchronous way.
1580 * The result of "drain" is not directly handled by callers. Then,
1581 * if someone is calling drain, we don't have to call drain more.
1582 * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if
1583 * there is a race. We just do loose check here.
1584 */
1585 if (atomic_read(&memcg_drain_count))
1586 return;
1587 /* Notify other cpus that system-wide "drain" is running */
1588 atomic_inc(&memcg_drain_count);
1589 get_online_cpus();
1590 for_each_online_cpu(cpu) {
1591 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1592 schedule_work_on(cpu, &stock->work);
1593 }
1594 put_online_cpus();
1595 atomic_dec(&memcg_drain_count);
1596 /* We don't wait for flush_work */
1597}
1598
1599/* This is a synchronous drain interface. */
1600static void drain_all_stock_sync(void)
1601{
1602 /* called when force_empty is called */
1603 atomic_inc(&memcg_drain_count);
1604 schedule_on_each_cpu(drain_local_stock);
1605 atomic_dec(&memcg_drain_count);
1606}
1607
1608static int __cpuinit memcg_stock_cpu_callback(struct notifier_block *nb,
1609 unsigned long action,
1610 void *hcpu)
1611{
1612 int cpu = (unsigned long)hcpu;
1613 struct memcg_stock_pcp *stock;
1614
1615 if (action != CPU_DEAD)
1616 return NOTIFY_OK;
1617 stock = &per_cpu(memcg_stock, cpu);
1618 drain_stock(stock);
1619 return NOTIFY_OK;
1620}
1621
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001622
1623/* See __mem_cgroup_try_charge() for details */
1624enum {
1625 CHARGE_OK, /* success */
1626 CHARGE_RETRY, /* need to retry but retry is not bad */
1627 CHARGE_NOMEM, /* we can't do more. return -ENOMEM */
1628 CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */
1629 CHARGE_OOM_DIE, /* the current is killed because of OOM */
1630};
1631
1632static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
1633 int csize, bool oom_check)
1634{
1635 struct mem_cgroup *mem_over_limit;
1636 struct res_counter *fail_res;
1637 unsigned long flags = 0;
1638 int ret;
1639
1640 ret = res_counter_charge(&mem->res, csize, &fail_res);
1641
1642 if (likely(!ret)) {
1643 if (!do_swap_account)
1644 return CHARGE_OK;
1645 ret = res_counter_charge(&mem->memsw, csize, &fail_res);
1646 if (likely(!ret))
1647 return CHARGE_OK;
1648
1649 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
1650 flags |= MEM_CGROUP_RECLAIM_NOSWAP;
1651 } else
1652 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
1653
1654 if (csize > PAGE_SIZE) /* change csize and retry */
1655 return CHARGE_RETRY;
1656
1657 if (!(gfp_mask & __GFP_WAIT))
1658 return CHARGE_WOULDBLOCK;
1659
1660 ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
1661 gfp_mask, flags);
1662 /*
1663 * try_to_free_mem_cgroup_pages() might not give us a full
1664 * picture of reclaim. Some pages are reclaimed and might be
1665 * moved to swap cache or just unmapped from the cgroup.
1666 * Check the limit again to see if the reclaim reduced the
1667 * current usage of the cgroup before giving up
1668 */
1669 if (ret || mem_cgroup_check_under_limit(mem_over_limit))
1670 return CHARGE_RETRY;
1671
1672 /*
1673 * At task move, charge accounts can be doubly counted. So, it's
1674 * better to wait until the end of task_move if something is going on.
1675 */
1676 if (mem_cgroup_wait_acct_move(mem_over_limit))
1677 return CHARGE_RETRY;
1678
1679 /* If we don't need to call oom-killer at el, return immediately */
1680 if (!oom_check)
1681 return CHARGE_NOMEM;
1682 /* check OOM */
1683 if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask))
1684 return CHARGE_OOM_DIE;
1685
1686 return CHARGE_RETRY;
1687}
1688
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001689/*
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001690 * Unlike exported interface, "oom" parameter is added. if oom==true,
1691 * oom-killer can be invoked.
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001692 */
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001693static int __mem_cgroup_try_charge(struct mm_struct *mm,
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001694 gfp_t gfp_mask, struct mem_cgroup **memcg, bool oom)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001695{
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001696 int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
1697 struct mem_cgroup *mem = NULL;
1698 int ret;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001699 int csize = CHARGE_SIZE;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001700
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001701 /*
1702 * Unlike gloval-vm's OOM-kill, we're not in memory shortage
1703 * in system level. So, allow to go ahead dying process in addition to
1704 * MEMDIE process.
1705 */
1706 if (unlikely(test_thread_flag(TIF_MEMDIE)
1707 || fatal_signal_pending(current)))
1708 goto bypass;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001709
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001710 /*
Hugh Dickins3be91272008-02-07 00:14:19 -08001711 * We always charge the cgroup the mm_struct belongs to.
1712 * The mm_struct's mem_cgroup changes on task migration if the
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001713 * thread group leader migrates. It's possible that mm is not
1714 * set, if so charge the init_mm (happens for pagecache usage).
1715 */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001716 if (!*memcg && !mm)
1717 goto bypass;
1718again:
1719 if (*memcg) { /* css should be a valid one */
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001720 mem = *memcg;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001721 VM_BUG_ON(css_is_removed(&mem->css));
1722 if (mem_cgroup_is_root(mem))
1723 goto done;
1724 if (consume_stock(mem))
1725 goto done;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001726 css_get(&mem->css);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001727 } else {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001728 struct task_struct *p;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001729
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001730 rcu_read_lock();
1731 p = rcu_dereference(mm->owner);
1732 VM_BUG_ON(!p);
1733 /*
1734 * because we don't have task_lock(), "p" can exit while
1735 * we're here. In that case, "mem" can point to root
1736 * cgroup but never be NULL. (and task_struct itself is freed
1737 * by RCU, cgroup itself is RCU safe.) Then, we have small
1738 * risk here to get wrong cgroup. But such kind of mis-account
1739 * by race always happens because we don't have cgroup_mutex().
1740 * It's overkill and we allow that small race, here.
1741 */
1742 mem = mem_cgroup_from_task(p);
1743 VM_BUG_ON(!mem);
1744 if (mem_cgroup_is_root(mem)) {
1745 rcu_read_unlock();
1746 goto done;
1747 }
1748 if (consume_stock(mem)) {
1749 /*
1750 * It seems dagerous to access memcg without css_get().
1751 * But considering how consume_stok works, it's not
1752 * necessary. If consume_stock success, some charges
1753 * from this memcg are cached on this cpu. So, we
1754 * don't need to call css_get()/css_tryget() before
1755 * calling consume_stock().
1756 */
1757 rcu_read_unlock();
1758 goto done;
1759 }
1760 /* after here, we may be blocked. we need to get refcnt */
1761 if (!css_tryget(&mem->css)) {
1762 rcu_read_unlock();
1763 goto again;
1764 }
1765 rcu_read_unlock();
1766 }
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001767
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001768 do {
1769 bool oom_check;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001770
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001771 /* If killed, bypass charge */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001772 if (fatal_signal_pending(current)) {
1773 css_put(&mem->css);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001774 goto bypass;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001775 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001776
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001777 oom_check = false;
1778 if (oom && !nr_oom_retries) {
1779 oom_check = true;
1780 nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
1781 }
Balbir Singh6d61ef42009-01-07 18:08:06 -08001782
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001783 ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check);
1784
1785 switch (ret) {
1786 case CHARGE_OK:
1787 break;
1788 case CHARGE_RETRY: /* not in OOM situation but retry */
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001789 csize = PAGE_SIZE;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001790 css_put(&mem->css);
1791 mem = NULL;
1792 goto again;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001793 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001794 css_put(&mem->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001795 goto nomem;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001796 case CHARGE_NOMEM: /* OOM routine works */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001797 if (!oom) {
1798 css_put(&mem->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001799 goto nomem;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001800 }
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001801 /* If oom, we never return -ENOMEM */
1802 nr_oom_retries--;
1803 break;
1804 case CHARGE_OOM_DIE: /* Killed by OOM Killer */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001805 css_put(&mem->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001806 goto bypass;
Balbir Singh66e17072008-02-07 00:13:56 -08001807 }
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001808 } while (ret != CHARGE_OK);
1809
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001810 if (csize > PAGE_SIZE)
1811 refill_stock(mem, csize - PAGE_SIZE);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001812 css_put(&mem->css);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07001813done:
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001814 *memcg = mem;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001815 return 0;
1816nomem:
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001817 *memcg = NULL;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001818 return -ENOMEM;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001819bypass:
1820 *memcg = NULL;
1821 return 0;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001822}
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001823
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07001824/*
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08001825 * Somemtimes we have to undo a charge we got by try_charge().
1826 * This function is for that and do uncharge, put css's refcnt.
1827 * gotten by try_charge().
1828 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08001829static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
1830 unsigned long count)
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08001831{
1832 if (!mem_cgroup_is_root(mem)) {
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08001833 res_counter_uncharge(&mem->res, PAGE_SIZE * count);
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08001834 if (do_swap_account)
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08001835 res_counter_uncharge(&mem->memsw, PAGE_SIZE * count);
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08001836 }
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08001837}
1838
1839static void mem_cgroup_cancel_charge(struct mem_cgroup *mem)
1840{
1841 __mem_cgroup_cancel_charge(mem, 1);
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08001842}
1843
1844/*
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07001845 * A helper function to get mem_cgroup from ID. must be called under
1846 * rcu_read_lock(). The caller must check css_is_removed() or some if
1847 * it's concern. (dropping refcnt from swap can be called against removed
1848 * memcg.)
1849 */
1850static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
1851{
1852 struct cgroup_subsys_state *css;
1853
1854 /* ID 0 is unused ID */
1855 if (!id)
1856 return NULL;
1857 css = css_lookup(&mem_cgroup_subsys, id);
1858 if (!css)
1859 return NULL;
1860 return container_of(css, struct mem_cgroup, css);
1861}
1862
Wu Fengguange42d9d52009-12-16 12:19:59 +01001863struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08001864{
Wu Fengguange42d9d52009-12-16 12:19:59 +01001865 struct mem_cgroup *mem = NULL;
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07001866 struct page_cgroup *pc;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07001867 unsigned short id;
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08001868 swp_entry_t ent;
1869
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07001870 VM_BUG_ON(!PageLocked(page));
1871
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07001872 pc = lookup_page_cgroup(page);
Daisuke Nishimurac0bd3f62009-04-30 15:08:11 -07001873 lock_page_cgroup(pc);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07001874 if (PageCgroupUsed(pc)) {
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07001875 mem = pc->mem_cgroup;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07001876 if (mem && !css_tryget(&mem->css))
1877 mem = NULL;
Wu Fengguange42d9d52009-12-16 12:19:59 +01001878 } else if (PageSwapCache(page)) {
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07001879 ent.val = page_private(page);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07001880 id = lookup_swap_cgroup(ent);
1881 rcu_read_lock();
1882 mem = mem_cgroup_lookup(id);
1883 if (mem && !css_tryget(&mem->css))
1884 mem = NULL;
1885 rcu_read_unlock();
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07001886 }
Daisuke Nishimurac0bd3f62009-04-30 15:08:11 -07001887 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08001888 return mem;
1889}
1890
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001891/*
Daisuke Nishimuraa5e924f2009-01-07 18:08:28 -08001892 * commit a charge got by __mem_cgroup_try_charge() and makes page_cgroup to be
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001893 * USED state. If already USED, uncharge and return.
1894 */
1895
1896static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
1897 struct page_cgroup *pc,
1898 enum charge_type ctype)
1899{
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001900 /* try_charge() can return NULL to *memcg, taking care of it. */
1901 if (!mem)
1902 return;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07001903
1904 lock_page_cgroup(pc);
1905 if (unlikely(PageCgroupUsed(pc))) {
1906 unlock_page_cgroup(pc);
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08001907 mem_cgroup_cancel_charge(mem);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001908 return;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07001909 }
Balbir Singh4b3bde42009-09-23 15:56:32 -07001910
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001911 pc->mem_cgroup = mem;
KAMEZAWA Hiroyuki261fb612009-09-23 15:56:33 -07001912 /*
1913 * We access a page_cgroup asynchronously without lock_page_cgroup().
1914 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
1915 * is accessed after testing USED bit. To make pc->mem_cgroup visible
1916 * before USED bit, we need memory barrier here.
1917 * See mem_cgroup_add_lru_list(), etc.
1918 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001919 smp_wmb();
Balbir Singh4b3bde42009-09-23 15:56:32 -07001920 switch (ctype) {
1921 case MEM_CGROUP_CHARGE_TYPE_CACHE:
1922 case MEM_CGROUP_CHARGE_TYPE_SHMEM:
1923 SetPageCgroupCache(pc);
1924 SetPageCgroupUsed(pc);
1925 break;
1926 case MEM_CGROUP_CHARGE_TYPE_MAPPED:
1927 ClearPageCgroupCache(pc);
1928 SetPageCgroupUsed(pc);
1929 break;
1930 default:
1931 break;
1932 }
Hugh Dickins3be91272008-02-07 00:14:19 -08001933
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001934 mem_cgroup_charge_statistics(mem, pc, true);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07001935
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07001936 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki430e48632010-03-10 15:22:30 -08001937 /*
1938 * "charge_statistics" updated event counter. Then, check it.
1939 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
1940 * if they exceeds softlimit.
1941 */
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08001942 memcg_check_events(mem, pc->page);
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001943}
1944
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001945/**
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08001946 * __mem_cgroup_move_account - move account of the page
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001947 * @pc: page_cgroup of the page.
1948 * @from: mem_cgroup which the page is moved from.
1949 * @to: mem_cgroup which the page is moved to. @from != @to.
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08001950 * @uncharge: whether we should call uncharge and css_put against @from.
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001951 *
1952 * The caller must confirm following.
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001953 * - page is not on LRU (isolate_page() is useful.)
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08001954 * - the pc is locked, used, and ->mem_cgroup points to @from.
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001955 *
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08001956 * This function doesn't do "charge" nor css_get to new cgroup. It should be
1957 * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is
1958 * true, this function does "uncharge" from old cgroup, but it doesn't if
1959 * @uncharge is false, so a caller should do "uncharge".
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001960 */
1961
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08001962static void __mem_cgroup_move_account(struct page_cgroup *pc,
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08001963 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001964{
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001965 VM_BUG_ON(from == to);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001966 VM_BUG_ON(PageLRU(pc->page));
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08001967 VM_BUG_ON(!PageCgroupLocked(pc));
1968 VM_BUG_ON(!PageCgroupUsed(pc));
1969 VM_BUG_ON(pc->mem_cgroup != from);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001970
KAMEZAWA Hiroyuki8725d542010-04-06 14:35:05 -07001971 if (PageCgroupFileMapped(pc)) {
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08001972 /* Update mapped_file data for mem_cgroup */
1973 preempt_disable();
1974 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
1975 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
1976 preempt_enable();
Balbir Singhd69b0422009-06-17 16:26:34 -07001977 }
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08001978 mem_cgroup_charge_statistics(from, pc, false);
1979 if (uncharge)
1980 /* This is not "cancel", but cancel_charge does all we need. */
1981 mem_cgroup_cancel_charge(from);
Balbir Singhd69b0422009-06-17 16:26:34 -07001982
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08001983 /* caller should have done css_get */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001984 pc->mem_cgroup = to;
1985 mem_cgroup_charge_statistics(to, pc, true);
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07001986 /*
1987 * We charges against "to" which may not have any tasks. Then, "to"
1988 * can be under rmdir(). But in current implementation, caller of
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08001989 * this function is just force_empty() and move charge, so it's
1990 * garanteed that "to" is never removed. So, we don't check rmdir
1991 * status here.
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07001992 */
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08001993}
1994
1995/*
1996 * check whether the @pc is valid for moving account and call
1997 * __mem_cgroup_move_account()
1998 */
1999static int mem_cgroup_move_account(struct page_cgroup *pc,
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002000 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08002001{
2002 int ret = -EINVAL;
2003 lock_page_cgroup(pc);
2004 if (PageCgroupUsed(pc) && pc->mem_cgroup == from) {
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002005 __mem_cgroup_move_account(pc, from, to, uncharge);
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08002006 ret = 0;
2007 }
2008 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08002009 /*
2010 * check events
2011 */
2012 memcg_check_events(to, pc->page);
2013 memcg_check_events(from, pc->page);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002014 return ret;
2015}
2016
2017/*
2018 * move charges to its parent.
2019 */
2020
2021static int mem_cgroup_move_parent(struct page_cgroup *pc,
2022 struct mem_cgroup *child,
2023 gfp_t gfp_mask)
2024{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002025 struct page *page = pc->page;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002026 struct cgroup *cg = child->css.cgroup;
2027 struct cgroup *pcg = cg->parent;
2028 struct mem_cgroup *parent;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002029 int ret;
2030
2031 /* Is ROOT ? */
2032 if (!pcg)
2033 return -EINVAL;
2034
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08002035 ret = -EBUSY;
2036 if (!get_page_unless_zero(page))
2037 goto out;
2038 if (isolate_lru_page(page))
2039 goto put;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002040
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002041 parent = mem_cgroup_from_cont(pcg);
KAMEZAWA Hiroyuki430e48632010-03-10 15:22:30 -08002042 ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false);
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002043 if (ret || !parent)
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08002044 goto put_back;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002045
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002046 ret = mem_cgroup_move_account(pc, child, parent, true);
2047 if (ret)
2048 mem_cgroup_cancel_charge(parent);
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08002049put_back:
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002050 putback_lru_page(page);
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08002051put:
Daisuke Nishimura40d58132009-01-15 13:51:12 -08002052 put_page(page);
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08002053out:
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002054 return ret;
2055}
2056
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002057/*
2058 * Charge the memory controller for page usage.
2059 * Return
2060 * 0 if the charge was successful
2061 * < 0 if the cgroup is over its limit
2062 */
2063static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002064 gfp_t gfp_mask, enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002065{
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002066 struct mem_cgroup *mem = NULL;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002067 struct page_cgroup *pc;
2068 int ret;
2069
2070 pc = lookup_page_cgroup(page);
2071 /* can happen at boot */
2072 if (unlikely(!pc))
2073 return 0;
2074 prefetchw(pc);
2075
KAMEZAWA Hiroyuki430e48632010-03-10 15:22:30 -08002076 ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true);
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002077 if (ret || !mem)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002078 return ret;
2079
2080 __mem_cgroup_commit_charge(mem, pc, ctype);
2081 return 0;
2082}
2083
2084int mem_cgroup_newpage_charge(struct page *page,
2085 struct mm_struct *mm, gfp_t gfp_mask)
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08002086{
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08002087 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07002088 return 0;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002089 if (PageCompound(page))
2090 return 0;
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002091 /*
2092 * If already mapped, we don't have to account.
2093 * If page cache, page->mapping has address_space.
2094 * But page->mapping may have out-of-use anon_vma pointer,
2095 * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
2096 * is NULL.
2097 */
2098 if (page_mapped(page) || (page->mapping && !PageAnon(page)))
2099 return 0;
2100 if (unlikely(!mm))
2101 mm = &init_mm;
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08002102 return mem_cgroup_charge_common(page, mm, gfp_mask,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002103 MEM_CGROUP_CHARGE_TYPE_MAPPED);
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08002104}
2105
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002106static void
2107__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2108 enum charge_type ctype);
2109
Balbir Singhe1a1cd52008-02-07 00:14:02 -08002110int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
2111 gfp_t gfp_mask)
Balbir Singh8697d332008-02-07 00:13:59 -08002112{
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002113 int ret;
2114
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08002115 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07002116 return 0;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002117 if (PageCompound(page))
2118 return 0;
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002119 /*
2120 * Corner case handling. This is called from add_to_page_cache()
2121 * in usual. But some FS (shmem) precharges this page before calling it
2122 * and call add_to_page_cache() with GFP_NOWAIT.
2123 *
2124 * For GFP_NOWAIT case, the page may be pre-charged before calling
2125 * add_to_page_cache(). (See shmem.c) check it here and avoid to call
2126 * charge twice. (It works but has to pay a bit larger cost.)
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002127 * And when the page is SwapCache, it should take swap information
2128 * into account. This is under lock_page() now.
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002129 */
2130 if (!(gfp_mask & __GFP_WAIT)) {
2131 struct page_cgroup *pc;
2132
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002133 pc = lookup_page_cgroup(page);
2134 if (!pc)
2135 return 0;
2136 lock_page_cgroup(pc);
2137 if (PageCgroupUsed(pc)) {
2138 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002139 return 0;
2140 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002141 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002142 }
2143
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002144 if (unlikely(!mm))
Balbir Singh8697d332008-02-07 00:13:59 -08002145 mm = &init_mm;
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002146
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -07002147 if (page_is_file_cache(page))
2148 return mem_cgroup_charge_common(page, mm, gfp_mask,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002149 MEM_CGROUP_CHARGE_TYPE_CACHE);
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002150
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002151 /* shmem */
2152 if (PageSwapCache(page)) {
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002153 struct mem_cgroup *mem = NULL;
2154
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002155 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2156 if (!ret)
2157 __mem_cgroup_commit_charge_swapin(page, mem,
2158 MEM_CGROUP_CHARGE_TYPE_SHMEM);
2159 } else
2160 ret = mem_cgroup_charge_common(page, mm, gfp_mask,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002161 MEM_CGROUP_CHARGE_TYPE_SHMEM);
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002162
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002163 return ret;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002164}
2165
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002166/*
2167 * While swap-in, try_charge -> commit or cancel, the page is locked.
2168 * And when try_charge() successfully returns, one refcnt to memcg without
Uwe Kleine-König21ae2952009-10-07 15:21:09 +02002169 * struct page_cgroup is acquired. This refcnt will be consumed by
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002170 * "commit()" or removed by "cancel()"
2171 */
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002172int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
2173 struct page *page,
2174 gfp_t mask, struct mem_cgroup **ptr)
2175{
2176 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002177 int ret;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002178
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08002179 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002180 return 0;
2181
2182 if (!do_swap_account)
2183 goto charge_cur_mm;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002184 /*
2185 * A racing thread's fault, or swapoff, may have already updated
Hugh Dickins407f9c82009-12-14 17:59:30 -08002186 * the pte, and even removed page from swap cache: in those cases
2187 * do_swap_page()'s pte_same() test will fail; but there's also a
2188 * KSM case which does need to charge the page.
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002189 */
2190 if (!PageSwapCache(page))
Hugh Dickins407f9c82009-12-14 17:59:30 -08002191 goto charge_cur_mm;
Wu Fengguange42d9d52009-12-16 12:19:59 +01002192 mem = try_get_mem_cgroup_from_page(page);
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002193 if (!mem)
2194 goto charge_cur_mm;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002195 *ptr = mem;
KAMEZAWA Hiroyuki430e48632010-03-10 15:22:30 -08002196 ret = __mem_cgroup_try_charge(NULL, mask, ptr, true);
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002197 css_put(&mem->css);
2198 return ret;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002199charge_cur_mm:
2200 if (unlikely(!mm))
2201 mm = &init_mm;
KAMEZAWA Hiroyuki430e48632010-03-10 15:22:30 -08002202 return __mem_cgroup_try_charge(mm, mask, ptr, true);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002203}
2204
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002205static void
2206__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2207 enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002208{
2209 struct page_cgroup *pc;
2210
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08002211 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002212 return;
2213 if (!ptr)
2214 return;
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002215 cgroup_exclude_rmdir(&ptr->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002216 pc = lookup_page_cgroup(page);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08002217 mem_cgroup_lru_del_before_commit_swapcache(page);
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002218 __mem_cgroup_commit_charge(ptr, pc, ctype);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08002219 mem_cgroup_lru_add_after_commit_swapcache(page);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002220 /*
2221 * Now swap is on-memory. This means this page may be
2222 * counted both as mem and swap....double count.
KAMEZAWA Hiroyuki03f3c432009-01-07 18:08:31 -08002223 * Fix it by uncharging from memsw. Basically, this SwapCache is stable
2224 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
2225 * may call delete_from_swap_cache() before reach here.
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002226 */
KAMEZAWA Hiroyuki03f3c432009-01-07 18:08:31 -08002227 if (do_swap_account && PageSwapCache(page)) {
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002228 swp_entry_t ent = {.val = page_private(page)};
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002229 unsigned short id;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002230 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002231
2232 id = swap_cgroup_record(ent, 0);
2233 rcu_read_lock();
2234 memcg = mem_cgroup_lookup(id);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002235 if (memcg) {
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002236 /*
2237 * This recorded memcg can be obsolete one. So, avoid
2238 * calling css_tryget
2239 */
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002240 if (!mem_cgroup_is_root(memcg))
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -07002241 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002242 mem_cgroup_swap_statistics(memcg, false);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002243 mem_cgroup_put(memcg);
2244 }
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002245 rcu_read_unlock();
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002246 }
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002247 /*
2248 * At swapin, we may charge account against cgroup which has no tasks.
2249 * So, rmdir()->pre_destroy() can be called while we do this charge.
2250 * In that case, we need to call pre_destroy() again. check it here.
2251 */
2252 cgroup_release_and_wakeup_rmdir(&ptr->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002253}
2254
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002255void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
2256{
2257 __mem_cgroup_commit_charge_swapin(page, ptr,
2258 MEM_CGROUP_CHARGE_TYPE_MAPPED);
2259}
2260
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002261void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
2262{
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08002263 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002264 return;
2265 if (!mem)
2266 return;
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002267 mem_cgroup_cancel_charge(mem);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002268}
2269
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002270static void
2271__do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype)
2272{
2273 struct memcg_batch_info *batch = NULL;
2274 bool uncharge_memsw = true;
2275 /* If swapout, usage of swap doesn't decrease */
2276 if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
2277 uncharge_memsw = false;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002278
2279 batch = &current->memcg_batch;
2280 /*
2281 * In usual, we do css_get() when we remember memcg pointer.
2282 * But in this case, we keep res->usage until end of a series of
2283 * uncharges. Then, it's ok to ignore memcg's refcnt.
2284 */
2285 if (!batch->memcg)
2286 batch->memcg = mem;
2287 /*
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002288 * do_batch > 0 when unmapping pages or inode invalidate/truncate.
2289 * In those cases, all pages freed continously can be expected to be in
2290 * the same cgroup and we have chance to coalesce uncharges.
2291 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
2292 * because we want to do uncharge as soon as possible.
2293 */
2294
2295 if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
2296 goto direct_uncharge;
2297
2298 /*
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002299 * In typical case, batch->memcg == mem. This means we can
2300 * merge a series of uncharges to an uncharge of res_counter.
2301 * If not, we uncharge res_counter ony by one.
2302 */
2303 if (batch->memcg != mem)
2304 goto direct_uncharge;
2305 /* remember freed charge and uncharge it later */
2306 batch->bytes += PAGE_SIZE;
2307 if (uncharge_memsw)
2308 batch->memsw_bytes += PAGE_SIZE;
2309 return;
2310direct_uncharge:
2311 res_counter_uncharge(&mem->res, PAGE_SIZE);
2312 if (uncharge_memsw)
2313 res_counter_uncharge(&mem->memsw, PAGE_SIZE);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002314 if (unlikely(batch->memcg != mem))
2315 memcg_oom_recover(mem);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002316 return;
2317}
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002318
Balbir Singh8697d332008-02-07 00:13:59 -08002319/*
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002320 * uncharge if !page_mapped(page)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002321 */
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002322static struct mem_cgroup *
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002323__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002324{
Hugh Dickins82895462008-03-04 14:29:08 -08002325 struct page_cgroup *pc;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002326 struct mem_cgroup *mem = NULL;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002327
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08002328 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002329 return NULL;
Balbir Singh40779602008-04-04 14:29:59 -07002330
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002331 if (PageSwapCache(page))
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002332 return NULL;
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002333
Balbir Singh8697d332008-02-07 00:13:59 -08002334 /*
Balbir Singh3c541e12008-02-07 00:14:41 -08002335 * Check if our page_cgroup is valid
Balbir Singh8697d332008-02-07 00:13:59 -08002336 */
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002337 pc = lookup_page_cgroup(page);
2338 if (unlikely(!pc || !PageCgroupUsed(pc)))
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002339 return NULL;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002340
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002341 lock_page_cgroup(pc);
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002342
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002343 mem = pc->mem_cgroup;
2344
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002345 if (!PageCgroupUsed(pc))
2346 goto unlock_out;
2347
2348 switch (ctype) {
2349 case MEM_CGROUP_CHARGE_TYPE_MAPPED:
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07002350 case MEM_CGROUP_CHARGE_TYPE_DROP:
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002351 /* See mem_cgroup_prepare_migration() */
2352 if (page_mapped(page) || PageCgroupMigration(pc))
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002353 goto unlock_out;
2354 break;
2355 case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
2356 if (!PageAnon(page)) { /* Shared memory */
2357 if (page->mapping && !page_is_file_cache(page))
2358 goto unlock_out;
2359 } else if (page_mapped(page)) /* Anon */
2360 goto unlock_out;
2361 break;
2362 default:
2363 break;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002364 }
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002365
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002366 mem_cgroup_charge_statistics(mem, pc, false);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07002367
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002368 ClearPageCgroupUsed(pc);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08002369 /*
2370 * pc->mem_cgroup is not cleared here. It will be accessed when it's
2371 * freed from LRU. This is safe because uncharged page is expected not
2372 * to be reused (freed soon). Exception is SwapCache, it's handled by
2373 * special functions.
2374 */
Hugh Dickinsb9c565d2008-03-04 14:29:11 -08002375
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002376 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002377 /*
2378 * even after unlock, we have mem->res.usage here and this memcg
2379 * will never be freed.
2380 */
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08002381 memcg_check_events(mem, page);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002382 if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
2383 mem_cgroup_swap_statistics(mem, true);
2384 mem_cgroup_get(mem);
2385 }
2386 if (!mem_cgroup_is_root(mem))
2387 __do_uncharge(mem, ctype);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08002388
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002389 return mem;
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002390
2391unlock_out:
2392 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002393 return NULL;
Balbir Singh3c541e12008-02-07 00:14:41 -08002394}
2395
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002396void mem_cgroup_uncharge_page(struct page *page)
2397{
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002398 /* early check. */
2399 if (page_mapped(page))
2400 return;
2401 if (page->mapping && !PageAnon(page))
2402 return;
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002403 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
2404}
2405
2406void mem_cgroup_uncharge_cache_page(struct page *page)
2407{
2408 VM_BUG_ON(page_mapped(page));
KAMEZAWA Hiroyukib7abea92008-10-18 20:28:09 -07002409 VM_BUG_ON(page->mapping);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002410 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
2411}
2412
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002413/*
2414 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
2415 * In that cases, pages are freed continuously and we can expect pages
2416 * are in the same memcg. All these calls itself limits the number of
2417 * pages freed at once, then uncharge_start/end() is called properly.
2418 * This may be called prural(2) times in a context,
2419 */
2420
2421void mem_cgroup_uncharge_start(void)
2422{
2423 current->memcg_batch.do_batch++;
2424 /* We can do nest. */
2425 if (current->memcg_batch.do_batch == 1) {
2426 current->memcg_batch.memcg = NULL;
2427 current->memcg_batch.bytes = 0;
2428 current->memcg_batch.memsw_bytes = 0;
2429 }
2430}
2431
2432void mem_cgroup_uncharge_end(void)
2433{
2434 struct memcg_batch_info *batch = &current->memcg_batch;
2435
2436 if (!batch->do_batch)
2437 return;
2438
2439 batch->do_batch--;
2440 if (batch->do_batch) /* If stacked, do nothing. */
2441 return;
2442
2443 if (!batch->memcg)
2444 return;
2445 /*
2446 * This "batch->memcg" is valid without any css_get/put etc...
2447 * bacause we hide charges behind us.
2448 */
2449 if (batch->bytes)
2450 res_counter_uncharge(&batch->memcg->res, batch->bytes);
2451 if (batch->memsw_bytes)
2452 res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002453 memcg_oom_recover(batch->memcg);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002454 /* forget this pointer (for sanity check) */
2455 batch->memcg = NULL;
2456}
2457
Daisuke Nishimurae767e052009-05-28 14:34:28 -07002458#ifdef CONFIG_SWAP
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002459/*
Daisuke Nishimurae767e052009-05-28 14:34:28 -07002460 * called after __delete_from_swap_cache() and drop "page" account.
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002461 * memcg information is recorded to swap_cgroup of "ent"
2462 */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07002463void
2464mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002465{
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002466 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07002467 int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002468
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07002469 if (!swapout) /* this was a swap cache but the swap is unused ! */
2470 ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
2471
2472 memcg = __mem_cgroup_uncharge_common(page, ctype);
2473
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002474 /*
2475 * record memcg information, if swapout && memcg != NULL,
2476 * mem_cgroup_get() was called in uncharge().
2477 */
2478 if (do_swap_account && swapout && memcg)
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002479 swap_cgroup_record(ent, css_id(&memcg->css));
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002480}
Daisuke Nishimurae767e052009-05-28 14:34:28 -07002481#endif
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002482
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002483#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
2484/*
2485 * called from swap_entry_free(). remove record in swap_cgroup and
2486 * uncharge "memsw" account.
2487 */
2488void mem_cgroup_uncharge_swap(swp_entry_t ent)
2489{
2490 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002491 unsigned short id;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002492
2493 if (!do_swap_account)
2494 return;
2495
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002496 id = swap_cgroup_record(ent, 0);
2497 rcu_read_lock();
2498 memcg = mem_cgroup_lookup(id);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002499 if (memcg) {
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002500 /*
2501 * We uncharge this because swap is freed.
2502 * This memcg can be obsolete one. We avoid calling css_tryget
2503 */
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002504 if (!mem_cgroup_is_root(memcg))
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -07002505 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002506 mem_cgroup_swap_statistics(memcg, false);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002507 mem_cgroup_put(memcg);
2508 }
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002509 rcu_read_unlock();
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002510}
Daisuke Nishimura02491442010-03-10 15:22:17 -08002511
2512/**
2513 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2514 * @entry: swap entry to be moved
2515 * @from: mem_cgroup which the entry is moved from
2516 * @to: mem_cgroup which the entry is moved to
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002517 * @need_fixup: whether we should fixup res_counters and refcounts.
Daisuke Nishimura02491442010-03-10 15:22:17 -08002518 *
2519 * It succeeds only when the swap_cgroup's record for this entry is the same
2520 * as the mem_cgroup's id of @from.
2521 *
2522 * Returns 0 on success, -EINVAL on failure.
2523 *
2524 * The caller must have charged to @to, IOW, called res_counter_charge() about
2525 * both res and memsw, and called css_get().
2526 */
2527static int mem_cgroup_move_swap_account(swp_entry_t entry,
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002528 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
Daisuke Nishimura02491442010-03-10 15:22:17 -08002529{
2530 unsigned short old_id, new_id;
2531
2532 old_id = css_id(&from->css);
2533 new_id = css_id(&to->css);
2534
2535 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08002536 mem_cgroup_swap_statistics(from, false);
Daisuke Nishimura02491442010-03-10 15:22:17 -08002537 mem_cgroup_swap_statistics(to, true);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002538 /*
2539 * This function is only called from task migration context now.
2540 * It postpones res_counter and refcount handling till the end
2541 * of task migration(mem_cgroup_clear_mc()) for performance
2542 * improvement. But we cannot postpone mem_cgroup_get(to)
2543 * because if the process that has been moved to @to does
2544 * swap-in, the refcount of @to might be decreased to 0.
2545 */
Daisuke Nishimura02491442010-03-10 15:22:17 -08002546 mem_cgroup_get(to);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002547 if (need_fixup) {
2548 if (!mem_cgroup_is_root(from))
2549 res_counter_uncharge(&from->memsw, PAGE_SIZE);
2550 mem_cgroup_put(from);
2551 /*
2552 * we charged both to->res and to->memsw, so we should
2553 * uncharge to->res.
2554 */
2555 if (!mem_cgroup_is_root(to))
2556 res_counter_uncharge(&to->res, PAGE_SIZE);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002557 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08002558 return 0;
2559 }
2560 return -EINVAL;
2561}
2562#else
2563static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002564 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
Daisuke Nishimura02491442010-03-10 15:22:17 -08002565{
2566 return -EINVAL;
2567}
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002568#endif
2569
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08002570/*
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002571 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
2572 * page belongs to.
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08002573 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002574int mem_cgroup_prepare_migration(struct page *page,
2575 struct page *newpage, struct mem_cgroup **ptr)
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08002576{
2577 struct page_cgroup *pc;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002578 struct mem_cgroup *mem = NULL;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002579 enum charge_type ctype;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002580 int ret = 0;
Hugh Dickins8869b8f2008-03-04 14:29:09 -08002581
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08002582 if (mem_cgroup_disabled())
Balbir Singh40779602008-04-04 14:29:59 -07002583 return 0;
2584
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002585 pc = lookup_page_cgroup(page);
2586 lock_page_cgroup(pc);
2587 if (PageCgroupUsed(pc)) {
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002588 mem = pc->mem_cgroup;
2589 css_get(&mem->css);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002590 /*
2591 * At migrating an anonymous page, its mapcount goes down
2592 * to 0 and uncharge() will be called. But, even if it's fully
2593 * unmapped, migration may fail and this page has to be
2594 * charged again. We set MIGRATION flag here and delay uncharge
2595 * until end_migration() is called
2596 *
2597 * Corner Case Thinking
2598 * A)
2599 * When the old page was mapped as Anon and it's unmap-and-freed
2600 * while migration was ongoing.
2601 * If unmap finds the old page, uncharge() of it will be delayed
2602 * until end_migration(). If unmap finds a new page, it's
2603 * uncharged when it make mapcount to be 1->0. If unmap code
2604 * finds swap_migration_entry, the new page will not be mapped
2605 * and end_migration() will find it(mapcount==0).
2606 *
2607 * B)
2608 * When the old page was mapped but migraion fails, the kernel
2609 * remaps it. A charge for it is kept by MIGRATION flag even
2610 * if mapcount goes down to 0. We can do remap successfully
2611 * without charging it again.
2612 *
2613 * C)
2614 * The "old" page is under lock_page() until the end of
2615 * migration, so, the old page itself will not be swapped-out.
2616 * If the new page is swapped out before end_migraton, our
2617 * hook to usual swap-out path will catch the event.
2618 */
2619 if (PageAnon(page))
2620 SetPageCgroupMigration(pc);
Hugh Dickinsb9c565d2008-03-04 14:29:11 -08002621 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002622 unlock_page_cgroup(pc);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002623 /*
2624 * If the page is not charged at this point,
2625 * we return here.
2626 */
2627 if (!mem)
2628 return 0;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002629
Andrea Arcangeli93d5c9b2010-04-23 13:17:39 -04002630 *ptr = mem;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002631 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, ptr, false);
2632 css_put(&mem->css);/* drop extra refcnt */
2633 if (ret || *ptr == NULL) {
2634 if (PageAnon(page)) {
2635 lock_page_cgroup(pc);
2636 ClearPageCgroupMigration(pc);
2637 unlock_page_cgroup(pc);
2638 /*
2639 * The old page may be fully unmapped while we kept it.
2640 */
2641 mem_cgroup_uncharge_page(page);
2642 }
2643 return -ENOMEM;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002644 }
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002645 /*
2646 * We charge new page before it's used/mapped. So, even if unlock_page()
2647 * is called before end_migration, we can catch all events on this new
2648 * page. In the case new page is migrated but not remapped, new page's
2649 * mapcount will be finally 0 and we call uncharge in end_migration().
2650 */
2651 pc = lookup_page_cgroup(newpage);
2652 if (PageAnon(page))
2653 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
2654 else if (page_is_file_cache(page))
2655 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
2656 else
2657 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
2658 __mem_cgroup_commit_charge(mem, pc, ctype);
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002659 return ret;
2660}
Hugh Dickinsfb59e9f2008-03-04 14:29:16 -08002661
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002662/* remove redundant charge if migration failed*/
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002663void mem_cgroup_end_migration(struct mem_cgroup *mem,
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002664 struct page *oldpage, struct page *newpage)
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002665{
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002666 struct page *used, *unused;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002667 struct page_cgroup *pc;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002668
2669 if (!mem)
2670 return;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002671 /* blocks rmdir() */
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002672 cgroup_exclude_rmdir(&mem->css);
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002673 /* at migration success, oldpage->mapping is NULL. */
2674 if (oldpage->mapping) {
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002675 used = oldpage;
2676 unused = newpage;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002677 } else {
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002678 used = newpage;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002679 unused = oldpage;
2680 }
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002681 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002682 * We disallowed uncharge of pages under migration because mapcount
2683 * of the page goes down to zero, temporarly.
2684 * Clear the flag and check the page should be charged.
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002685 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002686 pc = lookup_page_cgroup(oldpage);
2687 lock_page_cgroup(pc);
2688 ClearPageCgroupMigration(pc);
2689 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002690
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002691 __mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE);
2692
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002693 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002694 * If a page is a file cache, radix-tree replacement is very atomic
2695 * and we can skip this check. When it was an Anon page, its mapcount
2696 * goes down to 0. But because we added MIGRATION flage, it's not
2697 * uncharged yet. There are several case but page->mapcount check
2698 * and USED bit check in mem_cgroup_uncharge_page() will do enough
2699 * check. (see prepare_charge() also)
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002700 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002701 if (PageAnon(used))
2702 mem_cgroup_uncharge_page(used);
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002703 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002704 * At migration, we may charge account against cgroup which has no
2705 * tasks.
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002706 * So, rmdir()->pre_destroy() can be called while we do this charge.
2707 * In that case, we need to call pre_destroy() again. check it here.
2708 */
2709 cgroup_release_and_wakeup_rmdir(&mem->css);
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08002710}
Pavel Emelianov78fb7462008-02-07 00:13:51 -08002711
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08002712/*
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002713 * A call to try to shrink memory usage on charge failure at shmem's swapin.
2714 * Calling hierarchical_reclaim is not enough because we should update
2715 * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
2716 * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
2717 * not from the memcg which this page would be charged to.
2718 * try_charge_swapin does all of these works properly.
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002719 */
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002720int mem_cgroup_shmem_charge_fallback(struct page *page,
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002721 struct mm_struct *mm,
2722 gfp_t gfp_mask)
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002723{
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002724 struct mem_cgroup *mem = NULL;
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002725 int ret;
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002726
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08002727 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07002728 return 0;
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002729
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002730 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2731 if (!ret)
2732 mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002733
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002734 return ret;
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002735}
2736
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002737static DEFINE_MUTEX(set_limit_mutex);
2738
KOSAKI Motohirod38d2a72009-01-06 14:39:44 -08002739static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002740 unsigned long long val)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002741{
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07002742 int retry_count;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002743 u64 memswlimit, memlimit;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002744 int ret = 0;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07002745 int children = mem_cgroup_count_children(memcg);
2746 u64 curusage, oldusage;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002747 int enlarge;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07002748
2749 /*
2750 * For keeping hierarchical_reclaim simple, how long we should retry
2751 * is depends on callers. We set our retry-count to be function
2752 * of # of children which we should visit in this loop.
2753 */
2754 retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
2755
2756 oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002757
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002758 enlarge = 0;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002759 while (retry_count) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002760 if (signal_pending(current)) {
2761 ret = -EINTR;
2762 break;
2763 }
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002764 /*
2765 * Rather than hide all in some function, I do this in
2766 * open coded manner. You see what this really does.
2767 * We have to guarantee mem->res.limit < mem->memsw.limit.
2768 */
2769 mutex_lock(&set_limit_mutex);
2770 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
2771 if (memswlimit < val) {
2772 ret = -EINVAL;
2773 mutex_unlock(&set_limit_mutex);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002774 break;
2775 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002776
2777 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
2778 if (memlimit < val)
2779 enlarge = 1;
2780
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002781 ret = res_counter_set_limit(&memcg->res, val);
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07002782 if (!ret) {
2783 if (memswlimit == val)
2784 memcg->memsw_is_minimum = true;
2785 else
2786 memcg->memsw_is_minimum = false;
2787 }
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002788 mutex_unlock(&set_limit_mutex);
2789
2790 if (!ret)
2791 break;
2792
Bob Liuaa20d482009-12-15 16:47:14 -08002793 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
Balbir Singh4e416952009-09-23 15:56:39 -07002794 MEM_CGROUP_RECLAIM_SHRINK);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07002795 curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
2796 /* Usage is reduced ? */
2797 if (curusage >= oldusage)
2798 retry_count--;
2799 else
2800 oldusage = curusage;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002801 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002802 if (!ret && enlarge)
2803 memcg_oom_recover(memcg);
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08002804
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002805 return ret;
2806}
2807
Li Zefan338c8432009-06-17 16:27:15 -07002808static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
2809 unsigned long long val)
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002810{
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07002811 int retry_count;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002812 u64 memlimit, memswlimit, oldusage, curusage;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07002813 int children = mem_cgroup_count_children(memcg);
2814 int ret = -EBUSY;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002815 int enlarge = 0;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002816
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07002817 /* see mem_cgroup_resize_res_limit */
2818 retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
2819 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002820 while (retry_count) {
2821 if (signal_pending(current)) {
2822 ret = -EINTR;
2823 break;
2824 }
2825 /*
2826 * Rather than hide all in some function, I do this in
2827 * open coded manner. You see what this really does.
2828 * We have to guarantee mem->res.limit < mem->memsw.limit.
2829 */
2830 mutex_lock(&set_limit_mutex);
2831 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
2832 if (memlimit > val) {
2833 ret = -EINVAL;
2834 mutex_unlock(&set_limit_mutex);
2835 break;
2836 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002837 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
2838 if (memswlimit < val)
2839 enlarge = 1;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002840 ret = res_counter_set_limit(&memcg->memsw, val);
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07002841 if (!ret) {
2842 if (memlimit == val)
2843 memcg->memsw_is_minimum = true;
2844 else
2845 memcg->memsw_is_minimum = false;
2846 }
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002847 mutex_unlock(&set_limit_mutex);
2848
2849 if (!ret)
2850 break;
2851
Balbir Singh4e416952009-09-23 15:56:39 -07002852 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
Balbir Singh75822b42009-09-23 15:56:38 -07002853 MEM_CGROUP_RECLAIM_NOSWAP |
2854 MEM_CGROUP_RECLAIM_SHRINK);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002855 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07002856 /* Usage is reduced ? */
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002857 if (curusage >= oldusage)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002858 retry_count--;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07002859 else
2860 oldusage = curusage;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002861 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002862 if (!ret && enlarge)
2863 memcg_oom_recover(memcg);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002864 return ret;
2865}
2866
Balbir Singh4e416952009-09-23 15:56:39 -07002867unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
KOSAKI Motohiro00918b62010-08-10 18:03:05 -07002868 gfp_t gfp_mask)
Balbir Singh4e416952009-09-23 15:56:39 -07002869{
2870 unsigned long nr_reclaimed = 0;
2871 struct mem_cgroup_per_zone *mz, *next_mz = NULL;
2872 unsigned long reclaimed;
2873 int loop = 0;
2874 struct mem_cgroup_tree_per_zone *mctz;
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07002875 unsigned long long excess;
Balbir Singh4e416952009-09-23 15:56:39 -07002876
2877 if (order > 0)
2878 return 0;
2879
KOSAKI Motohiro00918b62010-08-10 18:03:05 -07002880 mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
Balbir Singh4e416952009-09-23 15:56:39 -07002881 /*
2882 * This loop can run a while, specially if mem_cgroup's continuously
2883 * keep exceeding their soft limit and putting the system under
2884 * pressure
2885 */
2886 do {
2887 if (next_mz)
2888 mz = next_mz;
2889 else
2890 mz = mem_cgroup_largest_soft_limit_node(mctz);
2891 if (!mz)
2892 break;
2893
2894 reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
2895 gfp_mask,
2896 MEM_CGROUP_RECLAIM_SOFT);
2897 nr_reclaimed += reclaimed;
2898 spin_lock(&mctz->lock);
2899
2900 /*
2901 * If we failed to reclaim anything from this memory cgroup
2902 * it is time to move on to the next cgroup
2903 */
2904 next_mz = NULL;
2905 if (!reclaimed) {
2906 do {
2907 /*
2908 * Loop until we find yet another one.
2909 *
2910 * By the time we get the soft_limit lock
2911 * again, someone might have aded the
2912 * group back on the RB tree. Iterate to
2913 * make sure we get a different mem.
2914 * mem_cgroup_largest_soft_limit_node returns
2915 * NULL if no other cgroup is present on
2916 * the tree
2917 */
2918 next_mz =
2919 __mem_cgroup_largest_soft_limit_node(mctz);
2920 if (next_mz == mz) {
2921 css_put(&next_mz->mem->css);
2922 next_mz = NULL;
2923 } else /* next_mz == NULL or other memcg */
2924 break;
2925 } while (1);
2926 }
Balbir Singh4e416952009-09-23 15:56:39 -07002927 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07002928 excess = res_counter_soft_limit_excess(&mz->mem->res);
Balbir Singh4e416952009-09-23 15:56:39 -07002929 /*
2930 * One school of thought says that we should not add
2931 * back the node to the tree if reclaim returns 0.
2932 * But our reclaim could return 0, simply because due
2933 * to priority we are exposing a smaller subset of
2934 * memory to reclaim from. Consider this as a longer
2935 * term TODO.
2936 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07002937 /* If excess == 0, no tree ops */
2938 __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
Balbir Singh4e416952009-09-23 15:56:39 -07002939 spin_unlock(&mctz->lock);
2940 css_put(&mz->mem->css);
2941 loop++;
2942 /*
2943 * Could not reclaim anything and there are no more
2944 * mem cgroups to try or we seem to be looping without
2945 * reclaiming anything.
2946 */
2947 if (!nr_reclaimed &&
2948 (next_mz == NULL ||
2949 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
2950 break;
2951 } while (!nr_reclaimed);
2952 if (next_mz)
2953 css_put(&next_mz->mem->css);
2954 return nr_reclaimed;
2955}
2956
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002957/*
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08002958 * This routine traverse page_cgroup in given list and drop them all.
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08002959 * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
2960 */
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002961static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002962 int node, int zid, enum lru_list lru)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08002963{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002964 struct zone *zone;
2965 struct mem_cgroup_per_zone *mz;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002966 struct page_cgroup *pc, *busy;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002967 unsigned long flags, loop;
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -08002968 struct list_head *list;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002969 int ret = 0;
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -08002970
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002971 zone = &NODE_DATA(node)->node_zones[zid];
2972 mz = mem_cgroup_zoneinfo(mem, node, zid);
Christoph Lameterb69408e2008-10-18 20:26:14 -07002973 list = &mz->lists[lru];
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08002974
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002975 loop = MEM_CGROUP_ZSTAT(mz, lru);
2976 /* give some margin against EBUSY etc...*/
2977 loop += 256;
2978 busy = NULL;
2979 while (loop--) {
2980 ret = 0;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002981 spin_lock_irqsave(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002982 if (list_empty(list)) {
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002983 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002984 break;
2985 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002986 pc = list_entry(list->prev, struct page_cgroup, lru);
2987 if (busy == pc) {
2988 list_move(&pc->lru, list);
Thiago Farina648bcc72010-03-05 13:42:04 -08002989 busy = NULL;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002990 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002991 continue;
2992 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002993 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002994
KAMEZAWA Hiroyuki2c26fdd2009-01-07 18:08:10 -08002995 ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002996 if (ret == -ENOMEM)
2997 break;
2998
2999 if (ret == -EBUSY || ret == -EINVAL) {
3000 /* found lock contention or "pc" is obsolete. */
3001 busy = pc;
3002 cond_resched();
3003 } else
3004 busy = NULL;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003005 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003006
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003007 if (!ret && !list_empty(list))
3008 return -EBUSY;
3009 return ret;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003010}
3011
3012/*
3013 * make mem_cgroup's charge to be 0 if there is no task.
3014 * This enables deleting this mem_cgroup.
3015 */
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003016static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003017{
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003018 int ret;
3019 int node, zid, shrink;
3020 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003021 struct cgroup *cgrp = mem->css.cgroup;
Hugh Dickins8869b8f2008-03-04 14:29:09 -08003022
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003023 css_get(&mem->css);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003024
3025 shrink = 0;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003026 /* should free all ? */
3027 if (free_all)
3028 goto try_to_free;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003029move_account:
Daisuke Nishimurafce66472010-01-15 17:01:30 -08003030 do {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003031 ret = -EBUSY;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003032 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003033 goto out;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003034 ret = -EINTR;
3035 if (signal_pending(current))
3036 goto out;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003037 /* This is for making all *used* pages to be on LRU. */
3038 lru_add_drain_all();
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08003039 drain_all_stock_sync();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003040 ret = 0;
KAMEZAWA Hiroyuki299b4ea2009-01-29 14:25:17 -08003041 for_each_node_state(node, N_HIGH_MEMORY) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003042 for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
Christoph Lameterb69408e2008-10-18 20:26:14 -07003043 enum lru_list l;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003044 for_each_lru(l) {
3045 ret = mem_cgroup_force_empty_list(mem,
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003046 node, zid, l);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003047 if (ret)
3048 break;
3049 }
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08003050 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003051 if (ret)
3052 break;
3053 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003054 memcg_oom_recover(mem);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003055 /* it seems parent cgroup doesn't have enough mem */
3056 if (ret == -ENOMEM)
3057 goto try_to_free;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003058 cond_resched();
Daisuke Nishimurafce66472010-01-15 17:01:30 -08003059 /* "ret" should also be checked to ensure all lists are empty. */
3060 } while (mem->res.usage > 0 || ret);
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003061out:
3062 css_put(&mem->css);
3063 return ret;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003064
3065try_to_free:
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003066 /* returns EBUSY if there is a task or if we come here twice. */
3067 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003068 ret = -EBUSY;
3069 goto out;
3070 }
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003071 /* we call try-to-free pages for make this cgroup empty */
3072 lru_add_drain_all();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003073 /* try to free all pages in this cgroup */
3074 shrink = 1;
3075 while (nr_retries && mem->res.usage > 0) {
3076 int progress;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003077
3078 if (signal_pending(current)) {
3079 ret = -EINTR;
3080 goto out;
3081 }
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003082 progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
3083 false, get_swappiness(mem));
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003084 if (!progress) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003085 nr_retries--;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003086 /* maybe some writeback is necessary */
Jens Axboe8aa7e842009-07-09 14:52:32 +02003087 congestion_wait(BLK_RW_ASYNC, HZ/10);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003088 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003089
3090 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003091 lru_add_drain();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003092 /* try move_account...there may be some *locked* pages. */
Daisuke Nishimurafce66472010-01-15 17:01:30 -08003093 goto move_account;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003094}
3095
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003096int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
3097{
3098 return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
3099}
3100
3101
Balbir Singh18f59ea2009-01-07 18:08:07 -08003102static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
3103{
3104 return mem_cgroup_from_cont(cont)->use_hierarchy;
3105}
3106
3107static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
3108 u64 val)
3109{
3110 int retval = 0;
3111 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
3112 struct cgroup *parent = cont->parent;
3113 struct mem_cgroup *parent_mem = NULL;
3114
3115 if (parent)
3116 parent_mem = mem_cgroup_from_cont(parent);
3117
3118 cgroup_lock();
3119 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02003120 * If parent's use_hierarchy is set, we can't make any modifications
Balbir Singh18f59ea2009-01-07 18:08:07 -08003121 * in the child subtrees. If it is unset, then the change can
3122 * occur, provided the current cgroup has no children.
3123 *
3124 * For the root cgroup, parent_mem is NULL, we allow value to be
3125 * set if there are no children.
3126 */
3127 if ((!parent_mem || !parent_mem->use_hierarchy) &&
3128 (val == 1 || val == 0)) {
3129 if (list_empty(&cont->children))
3130 mem->use_hierarchy = val;
3131 else
3132 retval = -EBUSY;
3133 } else
3134 retval = -EINVAL;
3135 cgroup_unlock();
3136
3137 return retval;
3138}
3139
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003140struct mem_cgroup_idx_data {
3141 s64 val;
3142 enum mem_cgroup_stat_index idx;
3143};
3144
3145static int
3146mem_cgroup_get_idx_stat(struct mem_cgroup *mem, void *data)
3147{
3148 struct mem_cgroup_idx_data *d = data;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003149 d->val += mem_cgroup_read_stat(mem, d->idx);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003150 return 0;
3151}
3152
3153static void
3154mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
3155 enum mem_cgroup_stat_index idx, s64 *val)
3156{
3157 struct mem_cgroup_idx_data d;
3158 d.idx = idx;
3159 d.val = 0;
3160 mem_cgroup_walk_tree(mem, &d, mem_cgroup_get_idx_stat);
3161 *val = d.val;
3162}
3163
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003164static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
3165{
3166 u64 idx_val, val;
3167
3168 if (!mem_cgroup_is_root(mem)) {
3169 if (!swap)
3170 return res_counter_read_u64(&mem->res, RES_USAGE);
3171 else
3172 return res_counter_read_u64(&mem->memsw, RES_USAGE);
3173 }
3174
3175 mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE, &idx_val);
3176 val = idx_val;
3177 mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS, &idx_val);
3178 val += idx_val;
3179
3180 if (swap) {
3181 mem_cgroup_get_recursive_idx_stat(mem,
3182 MEM_CGROUP_STAT_SWAPOUT, &idx_val);
3183 val += idx_val;
3184 }
3185
3186 return val << PAGE_SHIFT;
3187}
3188
Paul Menage2c3daa72008-04-29 00:59:58 -07003189static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003190{
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003191 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003192 u64 val;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003193 int type, name;
3194
3195 type = MEMFILE_TYPE(cft->private);
3196 name = MEMFILE_ATTR(cft->private);
3197 switch (type) {
3198 case _MEM:
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003199 if (name == RES_USAGE)
3200 val = mem_cgroup_usage(mem, false);
3201 else
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003202 val = res_counter_read_u64(&mem->res, name);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003203 break;
3204 case _MEMSWAP:
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003205 if (name == RES_USAGE)
3206 val = mem_cgroup_usage(mem, true);
3207 else
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003208 val = res_counter_read_u64(&mem->memsw, name);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003209 break;
3210 default:
3211 BUG();
3212 break;
3213 }
3214 return val;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003215}
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003216/*
3217 * The user of this function is...
3218 * RES_LIMIT.
3219 */
Paul Menage856c13a2008-07-25 01:47:04 -07003220static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
3221 const char *buffer)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003222{
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003223 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003224 int type, name;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003225 unsigned long long val;
3226 int ret;
3227
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003228 type = MEMFILE_TYPE(cft->private);
3229 name = MEMFILE_ATTR(cft->private);
3230 switch (name) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003231 case RES_LIMIT:
Balbir Singh4b3bde42009-09-23 15:56:32 -07003232 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3233 ret = -EINVAL;
3234 break;
3235 }
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003236 /* This function does all necessary parse...reuse it */
3237 ret = res_counter_memparse_write_strategy(buffer, &val);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003238 if (ret)
3239 break;
3240 if (type == _MEM)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003241 ret = mem_cgroup_resize_limit(memcg, val);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003242 else
3243 ret = mem_cgroup_resize_memsw_limit(memcg, val);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003244 break;
Balbir Singh296c81d2009-09-23 15:56:36 -07003245 case RES_SOFT_LIMIT:
3246 ret = res_counter_memparse_write_strategy(buffer, &val);
3247 if (ret)
3248 break;
3249 /*
3250 * For memsw, soft limits are hard to implement in terms
3251 * of semantics, for now, we support soft limits for
3252 * control without swap
3253 */
3254 if (type == _MEM)
3255 ret = res_counter_set_soft_limit(&memcg->res, val);
3256 else
3257 ret = -EINVAL;
3258 break;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003259 default:
3260 ret = -EINVAL; /* should be BUG() ? */
3261 break;
3262 }
3263 return ret;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003264}
3265
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08003266static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
3267 unsigned long long *mem_limit, unsigned long long *memsw_limit)
3268{
3269 struct cgroup *cgroup;
3270 unsigned long long min_limit, min_memsw_limit, tmp;
3271
3272 min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3273 min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3274 cgroup = memcg->css.cgroup;
3275 if (!memcg->use_hierarchy)
3276 goto out;
3277
3278 while (cgroup->parent) {
3279 cgroup = cgroup->parent;
3280 memcg = mem_cgroup_from_cont(cgroup);
3281 if (!memcg->use_hierarchy)
3282 break;
3283 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
3284 min_limit = min(min_limit, tmp);
3285 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3286 min_memsw_limit = min(min_memsw_limit, tmp);
3287 }
3288out:
3289 *mem_limit = min_limit;
3290 *memsw_limit = min_memsw_limit;
3291 return;
3292}
3293
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003294static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003295{
3296 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003297 int type, name;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003298
3299 mem = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003300 type = MEMFILE_TYPE(event);
3301 name = MEMFILE_ATTR(event);
3302 switch (name) {
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003303 case RES_MAX_USAGE:
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003304 if (type == _MEM)
3305 res_counter_reset_max(&mem->res);
3306 else
3307 res_counter_reset_max(&mem->memsw);
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003308 break;
3309 case RES_FAILCNT:
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003310 if (type == _MEM)
3311 res_counter_reset_failcnt(&mem->res);
3312 else
3313 res_counter_reset_failcnt(&mem->memsw);
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003314 break;
3315 }
Balbir Singhf64c3f52009-09-23 15:56:37 -07003316
Pavel Emelyanov85cc59d2008-04-29 01:00:20 -07003317 return 0;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003318}
3319
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003320static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
3321 struct cftype *cft)
3322{
3323 return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
3324}
3325
Daisuke Nishimura02491442010-03-10 15:22:17 -08003326#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003327static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
3328 struct cftype *cft, u64 val)
3329{
3330 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3331
3332 if (val >= (1 << NR_MOVE_TYPE))
3333 return -EINVAL;
3334 /*
3335 * We check this value several times in both in can_attach() and
3336 * attach(), so we need cgroup lock to prevent this value from being
3337 * inconsistent.
3338 */
3339 cgroup_lock();
3340 mem->move_charge_at_immigrate = val;
3341 cgroup_unlock();
3342
3343 return 0;
3344}
Daisuke Nishimura02491442010-03-10 15:22:17 -08003345#else
3346static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
3347 struct cftype *cft, u64 val)
3348{
3349 return -ENOSYS;
3350}
3351#endif
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003352
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003353
3354/* For read statistics */
3355enum {
3356 MCS_CACHE,
3357 MCS_RSS,
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -08003358 MCS_FILE_MAPPED,
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003359 MCS_PGPGIN,
3360 MCS_PGPGOUT,
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003361 MCS_SWAP,
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003362 MCS_INACTIVE_ANON,
3363 MCS_ACTIVE_ANON,
3364 MCS_INACTIVE_FILE,
3365 MCS_ACTIVE_FILE,
3366 MCS_UNEVICTABLE,
3367 NR_MCS_STAT,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003368};
3369
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003370struct mcs_total_stat {
3371 s64 stat[NR_MCS_STAT];
3372};
3373
3374struct {
3375 char *local_name;
3376 char *total_name;
3377} memcg_stat_strings[NR_MCS_STAT] = {
3378 {"cache", "total_cache"},
3379 {"rss", "total_rss"},
Balbir Singhd69b0422009-06-17 16:26:34 -07003380 {"mapped_file", "total_mapped_file"},
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003381 {"pgpgin", "total_pgpgin"},
3382 {"pgpgout", "total_pgpgout"},
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003383 {"swap", "total_swap"},
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003384 {"inactive_anon", "total_inactive_anon"},
3385 {"active_anon", "total_active_anon"},
3386 {"inactive_file", "total_inactive_file"},
3387 {"active_file", "total_active_file"},
3388 {"unevictable", "total_unevictable"}
3389};
3390
3391
3392static int mem_cgroup_get_local_stat(struct mem_cgroup *mem, void *data)
3393{
3394 struct mcs_total_stat *s = data;
3395 s64 val;
3396
3397 /* per cpu stat */
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003398 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003399 s->stat[MCS_CACHE] += val * PAGE_SIZE;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003400 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003401 s->stat[MCS_RSS] += val * PAGE_SIZE;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003402 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED);
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -08003403 s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003404 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003405 s->stat[MCS_PGPGIN] += val;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003406 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003407 s->stat[MCS_PGPGOUT] += val;
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003408 if (do_swap_account) {
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003409 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003410 s->stat[MCS_SWAP] += val * PAGE_SIZE;
3411 }
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003412
3413 /* per zone stat */
3414 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
3415 s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
3416 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
3417 s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
3418 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
3419 s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
3420 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
3421 s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
3422 val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
3423 s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
3424 return 0;
3425}
3426
3427static void
3428mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
3429{
3430 mem_cgroup_walk_tree(mem, s, mem_cgroup_get_local_stat);
3431}
3432
Paul Menagec64745c2008-04-29 01:00:02 -07003433static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
3434 struct cgroup_map_cb *cb)
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003435{
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003436 struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003437 struct mcs_total_stat mystat;
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003438 int i;
3439
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003440 memset(&mystat, 0, sizeof(mystat));
3441 mem_cgroup_get_local_stat(mem_cont, &mystat);
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003442
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003443 for (i = 0; i < NR_MCS_STAT; i++) {
3444 if (i == MCS_SWAP && !do_swap_account)
3445 continue;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003446 cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003447 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08003448
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003449 /* Hierarchical information */
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08003450 {
3451 unsigned long long limit, memsw_limit;
3452 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
3453 cb->fill(cb, "hierarchical_memory_limit", limit);
3454 if (do_swap_account)
3455 cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
3456 }
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003457
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003458 memset(&mystat, 0, sizeof(mystat));
3459 mem_cgroup_get_total_stat(mem_cont, &mystat);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003460 for (i = 0; i < NR_MCS_STAT; i++) {
3461 if (i == MCS_SWAP && !do_swap_account)
3462 continue;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003463 cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003464 }
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003465
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003466#ifdef CONFIG_DEBUG_VM
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08003467 cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003468
3469 {
3470 int nid, zid;
3471 struct mem_cgroup_per_zone *mz;
3472 unsigned long recent_rotated[2] = {0, 0};
3473 unsigned long recent_scanned[2] = {0, 0};
3474
3475 for_each_online_node(nid)
3476 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
3477 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
3478
3479 recent_rotated[0] +=
3480 mz->reclaim_stat.recent_rotated[0];
3481 recent_rotated[1] +=
3482 mz->reclaim_stat.recent_rotated[1];
3483 recent_scanned[0] +=
3484 mz->reclaim_stat.recent_scanned[0];
3485 recent_scanned[1] +=
3486 mz->reclaim_stat.recent_scanned[1];
3487 }
3488 cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
3489 cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
3490 cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
3491 cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
3492 }
3493#endif
3494
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003495 return 0;
3496}
3497
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003498static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
3499{
3500 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3501
3502 return get_swappiness(memcg);
3503}
3504
3505static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
3506 u64 val)
3507{
3508 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3509 struct mem_cgroup *parent;
Li Zefan068b38c2009-01-15 13:51:26 -08003510
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003511 if (val > 100)
3512 return -EINVAL;
3513
3514 if (cgrp->parent == NULL)
3515 return -EINVAL;
3516
3517 parent = mem_cgroup_from_cont(cgrp->parent);
Li Zefan068b38c2009-01-15 13:51:26 -08003518
3519 cgroup_lock();
3520
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003521 /* If under hierarchy, only empty-root can set this value */
3522 if ((parent->use_hierarchy) ||
Li Zefan068b38c2009-01-15 13:51:26 -08003523 (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
3524 cgroup_unlock();
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003525 return -EINVAL;
Li Zefan068b38c2009-01-15 13:51:26 -08003526 }
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003527
3528 spin_lock(&memcg->reclaim_param_lock);
3529 memcg->swappiness = val;
3530 spin_unlock(&memcg->reclaim_param_lock);
3531
Li Zefan068b38c2009-01-15 13:51:26 -08003532 cgroup_unlock();
3533
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003534 return 0;
3535}
3536
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003537static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3538{
3539 struct mem_cgroup_threshold_ary *t;
3540 u64 usage;
3541 int i;
3542
3543 rcu_read_lock();
3544 if (!swap)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003545 t = rcu_dereference(memcg->thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003546 else
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003547 t = rcu_dereference(memcg->memsw_thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003548
3549 if (!t)
3550 goto unlock;
3551
3552 usage = mem_cgroup_usage(memcg, swap);
3553
3554 /*
3555 * current_threshold points to threshold just below usage.
3556 * If it's not true, a threshold was crossed after last
3557 * call of __mem_cgroup_threshold().
3558 */
Phil Carmody5407a562010-05-26 14:42:42 -07003559 i = t->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003560
3561 /*
3562 * Iterate backward over array of thresholds starting from
3563 * current_threshold and check if a threshold is crossed.
3564 * If none of thresholds below usage is crossed, we read
3565 * only one element of the array here.
3566 */
3567 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3568 eventfd_signal(t->entries[i].eventfd, 1);
3569
3570 /* i = current_threshold + 1 */
3571 i++;
3572
3573 /*
3574 * Iterate forward over array of thresholds starting from
3575 * current_threshold+1 and check if a threshold is crossed.
3576 * If none of thresholds above usage is crossed, we read
3577 * only one element of the array here.
3578 */
3579 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3580 eventfd_signal(t->entries[i].eventfd, 1);
3581
3582 /* Update current_threshold */
Phil Carmody5407a562010-05-26 14:42:42 -07003583 t->current_threshold = i - 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003584unlock:
3585 rcu_read_unlock();
3586}
3587
3588static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3589{
3590 __mem_cgroup_threshold(memcg, false);
3591 if (do_swap_account)
3592 __mem_cgroup_threshold(memcg, true);
3593}
3594
3595static int compare_thresholds(const void *a, const void *b)
3596{
3597 const struct mem_cgroup_threshold *_a = a;
3598 const struct mem_cgroup_threshold *_b = b;
3599
3600 return _a->threshold - _b->threshold;
3601}
3602
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003603static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem, void *data)
3604{
3605 struct mem_cgroup_eventfd_list *ev;
3606
3607 list_for_each_entry(ev, &mem->oom_notify, list)
3608 eventfd_signal(ev->eventfd, 1);
3609 return 0;
3610}
3611
3612static void mem_cgroup_oom_notify(struct mem_cgroup *mem)
3613{
3614 mem_cgroup_walk_tree(mem, NULL, mem_cgroup_oom_notify_cb);
3615}
3616
3617static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
3618 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003619{
3620 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003621 struct mem_cgroup_thresholds *thresholds;
3622 struct mem_cgroup_threshold_ary *new;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003623 int type = MEMFILE_TYPE(cft->private);
3624 u64 threshold, usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003625 int i, size, ret;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003626
3627 ret = res_counter_memparse_write_strategy(args, &threshold);
3628 if (ret)
3629 return ret;
3630
3631 mutex_lock(&memcg->thresholds_lock);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003632
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003633 if (type == _MEM)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003634 thresholds = &memcg->thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003635 else if (type == _MEMSWAP)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003636 thresholds = &memcg->memsw_thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003637 else
3638 BUG();
3639
3640 usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
3641
3642 /* Check if a threshold crossed before adding a new one */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003643 if (thresholds->primary)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003644 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3645
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003646 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003647
3648 /* Allocate memory for new array of thresholds */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003649 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003650 GFP_KERNEL);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003651 if (!new) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003652 ret = -ENOMEM;
3653 goto unlock;
3654 }
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003655 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003656
3657 /* Copy thresholds (if any) to new array */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003658 if (thresholds->primary) {
3659 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003660 sizeof(struct mem_cgroup_threshold));
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003661 }
3662
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003663 /* Add new threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003664 new->entries[size - 1].eventfd = eventfd;
3665 new->entries[size - 1].threshold = threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003666
3667 /* Sort thresholds. Registering of new threshold isn't time-critical */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003668 sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003669 compare_thresholds, NULL);
3670
3671 /* Find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003672 new->current_threshold = -1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003673 for (i = 0; i < size; i++) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003674 if (new->entries[i].threshold < usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003675 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003676 * new->current_threshold will not be used until
3677 * rcu_assign_pointer(), so it's safe to increment
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003678 * it here.
3679 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003680 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003681 }
3682 }
3683
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003684 /* Free old spare buffer and save old primary buffer as spare */
3685 kfree(thresholds->spare);
3686 thresholds->spare = thresholds->primary;
3687
3688 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003689
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003690 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003691 synchronize_rcu();
3692
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003693unlock:
3694 mutex_unlock(&memcg->thresholds_lock);
3695
3696 return ret;
3697}
3698
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003699static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003700 struct cftype *cft, struct eventfd_ctx *eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003701{
3702 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003703 struct mem_cgroup_thresholds *thresholds;
3704 struct mem_cgroup_threshold_ary *new;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003705 int type = MEMFILE_TYPE(cft->private);
3706 u64 usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003707 int i, j, size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003708
3709 mutex_lock(&memcg->thresholds_lock);
3710 if (type == _MEM)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003711 thresholds = &memcg->thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003712 else if (type == _MEMSWAP)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003713 thresholds = &memcg->memsw_thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003714 else
3715 BUG();
3716
3717 /*
3718 * Something went wrong if we trying to unregister a threshold
3719 * if we don't have thresholds
3720 */
3721 BUG_ON(!thresholds);
3722
3723 usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
3724
3725 /* Check if a threshold crossed before removing */
3726 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3727
3728 /* Calculate new number of threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003729 size = 0;
3730 for (i = 0; i < thresholds->primary->size; i++) {
3731 if (thresholds->primary->entries[i].eventfd != eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003732 size++;
3733 }
3734
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003735 new = thresholds->spare;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003736
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003737 /* Set thresholds array to NULL if we don't have thresholds */
3738 if (!size) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003739 kfree(new);
3740 new = NULL;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003741 goto swap_buffers;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003742 }
3743
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003744 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003745
3746 /* Copy thresholds and find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003747 new->current_threshold = -1;
3748 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3749 if (thresholds->primary->entries[i].eventfd == eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003750 continue;
3751
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003752 new->entries[j] = thresholds->primary->entries[i];
3753 if (new->entries[j].threshold < usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003754 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003755 * new->current_threshold will not be used
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003756 * until rcu_assign_pointer(), so it's safe to increment
3757 * it here.
3758 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003759 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003760 }
3761 j++;
3762 }
3763
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003764swap_buffers:
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003765 /* Swap primary and spare array */
3766 thresholds->spare = thresholds->primary;
3767 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003768
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003769 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003770 synchronize_rcu();
3771
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003772 mutex_unlock(&memcg->thresholds_lock);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003773}
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003774
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003775static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
3776 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
3777{
3778 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3779 struct mem_cgroup_eventfd_list *event;
3780 int type = MEMFILE_TYPE(cft->private);
3781
3782 BUG_ON(type != _OOM_TYPE);
3783 event = kmalloc(sizeof(*event), GFP_KERNEL);
3784 if (!event)
3785 return -ENOMEM;
3786
3787 mutex_lock(&memcg_oom_mutex);
3788
3789 event->eventfd = eventfd;
3790 list_add(&event->list, &memcg->oom_notify);
3791
3792 /* already in OOM ? */
3793 if (atomic_read(&memcg->oom_lock))
3794 eventfd_signal(eventfd, 1);
3795 mutex_unlock(&memcg_oom_mutex);
3796
3797 return 0;
3798}
3799
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003800static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003801 struct cftype *cft, struct eventfd_ctx *eventfd)
3802{
3803 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3804 struct mem_cgroup_eventfd_list *ev, *tmp;
3805 int type = MEMFILE_TYPE(cft->private);
3806
3807 BUG_ON(type != _OOM_TYPE);
3808
3809 mutex_lock(&memcg_oom_mutex);
3810
3811 list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) {
3812 if (ev->eventfd == eventfd) {
3813 list_del(&ev->list);
3814 kfree(ev);
3815 }
3816 }
3817
3818 mutex_unlock(&memcg_oom_mutex);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003819}
3820
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003821static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
3822 struct cftype *cft, struct cgroup_map_cb *cb)
3823{
3824 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3825
3826 cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable);
3827
3828 if (atomic_read(&mem->oom_lock))
3829 cb->fill(cb, "under_oom", 1);
3830 else
3831 cb->fill(cb, "under_oom", 0);
3832 return 0;
3833}
3834
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003835static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
3836 struct cftype *cft, u64 val)
3837{
3838 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3839 struct mem_cgroup *parent;
3840
3841 /* cannot set to root cgroup and only 0 and 1 are allowed */
3842 if (!cgrp->parent || !((val == 0) || (val == 1)))
3843 return -EINVAL;
3844
3845 parent = mem_cgroup_from_cont(cgrp->parent);
3846
3847 cgroup_lock();
3848 /* oom-kill-disable is a flag for subhierarchy. */
3849 if ((parent->use_hierarchy) ||
3850 (mem->use_hierarchy && !list_empty(&cgrp->children))) {
3851 cgroup_unlock();
3852 return -EINVAL;
3853 }
3854 mem->oom_kill_disable = val;
KAMEZAWA Hiroyuki4d845eb2010-06-29 15:05:18 -07003855 if (!val)
3856 memcg_oom_recover(mem);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003857 cgroup_unlock();
3858 return 0;
3859}
3860
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003861static struct cftype mem_cgroup_files[] = {
3862 {
Balbir Singh0eea1032008-02-07 00:13:57 -08003863 .name = "usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003864 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
Paul Menage2c3daa72008-04-29 00:59:58 -07003865 .read_u64 = mem_cgroup_read,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003866 .register_event = mem_cgroup_usage_register_event,
3867 .unregister_event = mem_cgroup_usage_unregister_event,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003868 },
3869 {
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003870 .name = "max_usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003871 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003872 .trigger = mem_cgroup_reset,
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003873 .read_u64 = mem_cgroup_read,
3874 },
3875 {
Balbir Singh0eea1032008-02-07 00:13:57 -08003876 .name = "limit_in_bytes",
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003877 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
Paul Menage856c13a2008-07-25 01:47:04 -07003878 .write_string = mem_cgroup_write,
Paul Menage2c3daa72008-04-29 00:59:58 -07003879 .read_u64 = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003880 },
3881 {
Balbir Singh296c81d2009-09-23 15:56:36 -07003882 .name = "soft_limit_in_bytes",
3883 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
3884 .write_string = mem_cgroup_write,
3885 .read_u64 = mem_cgroup_read,
3886 },
3887 {
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003888 .name = "failcnt",
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003889 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003890 .trigger = mem_cgroup_reset,
Paul Menage2c3daa72008-04-29 00:59:58 -07003891 .read_u64 = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003892 },
Balbir Singh8697d332008-02-07 00:13:59 -08003893 {
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003894 .name = "stat",
Paul Menagec64745c2008-04-29 01:00:02 -07003895 .read_map = mem_control_stat_show,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003896 },
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003897 {
3898 .name = "force_empty",
3899 .trigger = mem_cgroup_force_empty_write,
3900 },
Balbir Singh18f59ea2009-01-07 18:08:07 -08003901 {
3902 .name = "use_hierarchy",
3903 .write_u64 = mem_cgroup_hierarchy_write,
3904 .read_u64 = mem_cgroup_hierarchy_read,
3905 },
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003906 {
3907 .name = "swappiness",
3908 .read_u64 = mem_cgroup_swappiness_read,
3909 .write_u64 = mem_cgroup_swappiness_write,
3910 },
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003911 {
3912 .name = "move_charge_at_immigrate",
3913 .read_u64 = mem_cgroup_move_charge_read,
3914 .write_u64 = mem_cgroup_move_charge_write,
3915 },
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003916 {
3917 .name = "oom_control",
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003918 .read_map = mem_cgroup_oom_control_read,
3919 .write_u64 = mem_cgroup_oom_control_write,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003920 .register_event = mem_cgroup_oom_register_event,
3921 .unregister_event = mem_cgroup_oom_unregister_event,
3922 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
3923 },
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003924};
3925
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003926#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
3927static struct cftype memsw_cgroup_files[] = {
3928 {
3929 .name = "memsw.usage_in_bytes",
3930 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
3931 .read_u64 = mem_cgroup_read,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003932 .register_event = mem_cgroup_usage_register_event,
3933 .unregister_event = mem_cgroup_usage_unregister_event,
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003934 },
3935 {
3936 .name = "memsw.max_usage_in_bytes",
3937 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
3938 .trigger = mem_cgroup_reset,
3939 .read_u64 = mem_cgroup_read,
3940 },
3941 {
3942 .name = "memsw.limit_in_bytes",
3943 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
3944 .write_string = mem_cgroup_write,
3945 .read_u64 = mem_cgroup_read,
3946 },
3947 {
3948 .name = "memsw.failcnt",
3949 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
3950 .trigger = mem_cgroup_reset,
3951 .read_u64 = mem_cgroup_read,
3952 },
3953};
3954
3955static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
3956{
3957 if (!do_swap_account)
3958 return 0;
3959 return cgroup_add_files(cont, ss, memsw_cgroup_files,
3960 ARRAY_SIZE(memsw_cgroup_files));
3961};
3962#else
3963static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
3964{
3965 return 0;
3966}
3967#endif
3968
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08003969static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
3970{
3971 struct mem_cgroup_per_node *pn;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08003972 struct mem_cgroup_per_zone *mz;
Christoph Lameterb69408e2008-10-18 20:26:14 -07003973 enum lru_list l;
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07003974 int zone, tmp = node;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08003975 /*
3976 * This routine is called against possible nodes.
3977 * But it's BUG to call kmalloc() against offline node.
3978 *
3979 * TODO: this routine can waste much memory for nodes which will
3980 * never be onlined. It's better to use memory hotplug callback
3981 * function.
3982 */
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07003983 if (!node_state(node, N_NORMAL_MEMORY))
3984 tmp = -1;
3985 pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08003986 if (!pn)
3987 return 1;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08003988
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08003989 mem->info.nodeinfo[node] = pn;
3990 memset(pn, 0, sizeof(*pn));
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08003991
3992 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
3993 mz = &pn->zoneinfo[zone];
Christoph Lameterb69408e2008-10-18 20:26:14 -07003994 for_each_lru(l)
3995 INIT_LIST_HEAD(&mz->lists[l]);
Balbir Singhf64c3f52009-09-23 15:56:37 -07003996 mz->usage_in_excess = 0;
Balbir Singh4e416952009-09-23 15:56:39 -07003997 mz->on_tree = false;
3998 mz->mem = mem;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08003999 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004000 return 0;
4001}
4002
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004003static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
4004{
4005 kfree(mem->info.nodeinfo[node]);
4006}
4007
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004008static struct mem_cgroup *mem_cgroup_alloc(void)
4009{
4010 struct mem_cgroup *mem;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004011 int size = sizeof(struct mem_cgroup);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004012
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004013 /* Can be very big if MAX_NUMNODES is very big */
Jan Blunckc8dad2b2009-01-07 18:07:53 -08004014 if (size < PAGE_SIZE)
4015 mem = kmalloc(size, GFP_KERNEL);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004016 else
Jan Blunckc8dad2b2009-01-07 18:07:53 -08004017 mem = vmalloc(size);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004018
Dan Carpentere7bbcdf2010-03-23 13:35:12 -07004019 if (!mem)
4020 return NULL;
4021
4022 memset(mem, 0, size);
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004023 mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
4024 if (!mem->stat) {
4025 if (size < PAGE_SIZE)
4026 kfree(mem);
4027 else
4028 vfree(mem);
4029 mem = NULL;
4030 }
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004031 return mem;
4032}
4033
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004034/*
4035 * At destroying mem_cgroup, references from swap_cgroup can remain.
4036 * (scanning all at force_empty is too costly...)
4037 *
4038 * Instead of clearing all references at force_empty, we remember
4039 * the number of reference from swap_cgroup and free mem_cgroup when
4040 * it goes down to 0.
4041 *
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004042 * Removal of cgroup itself succeeds regardless of refs from swap.
4043 */
4044
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004045static void __mem_cgroup_free(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004046{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004047 int node;
4048
Balbir Singhf64c3f52009-09-23 15:56:37 -07004049 mem_cgroup_remove_from_trees(mem);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004050 free_css_id(&mem_cgroup_subsys, &mem->css);
4051
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004052 for_each_node_state(node, N_POSSIBLE)
4053 free_mem_cgroup_per_zone_info(mem, node);
4054
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004055 free_percpu(mem->stat);
4056 if (sizeof(struct mem_cgroup) < PAGE_SIZE)
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004057 kfree(mem);
4058 else
4059 vfree(mem);
4060}
4061
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004062static void mem_cgroup_get(struct mem_cgroup *mem)
4063{
4064 atomic_inc(&mem->refcnt);
4065}
4066
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004067static void __mem_cgroup_put(struct mem_cgroup *mem, int count)
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004068{
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004069 if (atomic_sub_and_test(count, &mem->refcnt)) {
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004070 struct mem_cgroup *parent = parent_mem_cgroup(mem);
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004071 __mem_cgroup_free(mem);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004072 if (parent)
4073 mem_cgroup_put(parent);
4074 }
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004075}
4076
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004077static void mem_cgroup_put(struct mem_cgroup *mem)
4078{
4079 __mem_cgroup_put(mem, 1);
4080}
4081
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004082/*
4083 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
4084 */
4085static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
4086{
4087 if (!mem->res.parent)
4088 return NULL;
4089 return mem_cgroup_from_res_counter(mem->res.parent, res);
4090}
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004091
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004092#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4093static void __init enable_swap_cgroup(void)
4094{
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08004095 if (!mem_cgroup_disabled() && really_do_swap_account)
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004096 do_swap_account = 1;
4097}
4098#else
4099static void __init enable_swap_cgroup(void)
4100{
4101}
4102#endif
4103
Balbir Singhf64c3f52009-09-23 15:56:37 -07004104static int mem_cgroup_soft_limit_tree_init(void)
4105{
4106 struct mem_cgroup_tree_per_node *rtpn;
4107 struct mem_cgroup_tree_per_zone *rtpz;
4108 int tmp, node, zone;
4109
4110 for_each_node_state(node, N_POSSIBLE) {
4111 tmp = node;
4112 if (!node_state(node, N_NORMAL_MEMORY))
4113 tmp = -1;
4114 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
4115 if (!rtpn)
4116 return 1;
4117
4118 soft_limit_tree.rb_tree_per_node[node] = rtpn;
4119
4120 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4121 rtpz = &rtpn->rb_tree_per_zone[zone];
4122 rtpz->rb_root = RB_ROOT;
4123 spin_lock_init(&rtpz->lock);
4124 }
4125 }
4126 return 0;
4127}
4128
Li Zefan0eb253e2009-01-15 13:51:25 -08004129static struct cgroup_subsys_state * __ref
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004130mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
4131{
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004132 struct mem_cgroup *mem, *parent;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004133 long error = -ENOMEM;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004134 int node;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004135
Jan Blunckc8dad2b2009-01-07 18:07:53 -08004136 mem = mem_cgroup_alloc();
4137 if (!mem)
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004138 return ERR_PTR(error);
Pavel Emelianov78fb7462008-02-07 00:13:51 -08004139
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004140 for_each_node_state(node, N_POSSIBLE)
4141 if (alloc_mem_cgroup_per_zone_info(mem, node))
4142 goto free_out;
Balbir Singhf64c3f52009-09-23 15:56:37 -07004143
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004144 /* root ? */
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004145 if (cont->parent == NULL) {
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08004146 int cpu;
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004147 enable_swap_cgroup();
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004148 parent = NULL;
Balbir Singh4b3bde42009-09-23 15:56:32 -07004149 root_mem_cgroup = mem;
Balbir Singhf64c3f52009-09-23 15:56:37 -07004150 if (mem_cgroup_soft_limit_tree_init())
4151 goto free_out;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08004152 for_each_possible_cpu(cpu) {
4153 struct memcg_stock_pcp *stock =
4154 &per_cpu(memcg_stock, cpu);
4155 INIT_WORK(&stock->work, drain_local_stock);
4156 }
4157 hotcpu_notifier(memcg_stock_cpu_callback, 0);
Balbir Singh18f59ea2009-01-07 18:08:07 -08004158 } else {
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004159 parent = mem_cgroup_from_cont(cont->parent);
Balbir Singh18f59ea2009-01-07 18:08:07 -08004160 mem->use_hierarchy = parent->use_hierarchy;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004161 mem->oom_kill_disable = parent->oom_kill_disable;
Balbir Singh18f59ea2009-01-07 18:08:07 -08004162 }
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004163
Balbir Singh18f59ea2009-01-07 18:08:07 -08004164 if (parent && parent->use_hierarchy) {
4165 res_counter_init(&mem->res, &parent->res);
4166 res_counter_init(&mem->memsw, &parent->memsw);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004167 /*
4168 * We increment refcnt of the parent to ensure that we can
4169 * safely access it on res_counter_charge/uncharge.
4170 * This refcnt will be decremented when freeing this
4171 * mem_cgroup(see mem_cgroup_put).
4172 */
4173 mem_cgroup_get(parent);
Balbir Singh18f59ea2009-01-07 18:08:07 -08004174 } else {
4175 res_counter_init(&mem->res, NULL);
4176 res_counter_init(&mem->memsw, NULL);
4177 }
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004178 mem->last_scanned_child = 0;
KOSAKI Motohiro2733c062009-01-07 18:08:23 -08004179 spin_lock_init(&mem->reclaim_param_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004180 INIT_LIST_HEAD(&mem->oom_notify);
Balbir Singh6d61ef42009-01-07 18:08:06 -08004181
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004182 if (parent)
4183 mem->swappiness = get_swappiness(parent);
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004184 atomic_set(&mem->refcnt, 1);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004185 mem->move_charge_at_immigrate = 0;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004186 mutex_init(&mem->thresholds_lock);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004187 return &mem->css;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004188free_out:
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004189 __mem_cgroup_free(mem);
Balbir Singh4b3bde42009-09-23 15:56:32 -07004190 root_mem_cgroup = NULL;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004191 return ERR_PTR(error);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004192}
4193
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07004194static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08004195 struct cgroup *cont)
4196{
4197 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07004198
4199 return mem_cgroup_force_empty(mem, false);
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08004200}
4201
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004202static void mem_cgroup_destroy(struct cgroup_subsys *ss,
4203 struct cgroup *cont)
4204{
Daisuke Nishimurac268e992009-01-15 13:51:13 -08004205 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
Daisuke Nishimurac268e992009-01-15 13:51:13 -08004206
Daisuke Nishimurac268e992009-01-15 13:51:13 -08004207 mem_cgroup_put(mem);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004208}
4209
4210static int mem_cgroup_populate(struct cgroup_subsys *ss,
4211 struct cgroup *cont)
4212{
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004213 int ret;
4214
4215 ret = cgroup_add_files(cont, ss, mem_cgroup_files,
4216 ARRAY_SIZE(mem_cgroup_files));
4217
4218 if (!ret)
4219 ret = register_memsw_files(cont, ss);
4220 return ret;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004221}
4222
Daisuke Nishimura02491442010-03-10 15:22:17 -08004223#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004224/* Handlers for move charge at task migration. */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004225#define PRECHARGE_COUNT_AT_ONCE 256
4226static int mem_cgroup_do_precharge(unsigned long count)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004227{
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004228 int ret = 0;
4229 int batch_count = PRECHARGE_COUNT_AT_ONCE;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004230 struct mem_cgroup *mem = mc.to;
4231
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004232 if (mem_cgroup_is_root(mem)) {
4233 mc.precharge += count;
4234 /* we don't need css_get for root */
4235 return ret;
4236 }
4237 /* try to charge at once */
4238 if (count > 1) {
4239 struct res_counter *dummy;
4240 /*
4241 * "mem" cannot be under rmdir() because we've already checked
4242 * by cgroup_lock_live_cgroup() that it is not removed and we
4243 * are still under the same cgroup_mutex. So we can postpone
4244 * css_get().
4245 */
4246 if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
4247 goto one_by_one;
4248 if (do_swap_account && res_counter_charge(&mem->memsw,
4249 PAGE_SIZE * count, &dummy)) {
4250 res_counter_uncharge(&mem->res, PAGE_SIZE * count);
4251 goto one_by_one;
4252 }
4253 mc.precharge += count;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004254 return ret;
4255 }
4256one_by_one:
4257 /* fall back to one by one charge */
4258 while (count--) {
4259 if (signal_pending(current)) {
4260 ret = -EINTR;
4261 break;
4262 }
4263 if (!batch_count--) {
4264 batch_count = PRECHARGE_COUNT_AT_ONCE;
4265 cond_resched();
4266 }
KAMEZAWA Hiroyuki430e48632010-03-10 15:22:30 -08004267 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004268 if (ret || !mem)
4269 /* mem_cgroup_clear_mc() will do uncharge later */
4270 return -ENOMEM;
4271 mc.precharge++;
4272 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004273 return ret;
4274}
4275
4276/**
4277 * is_target_pte_for_mc - check a pte whether it is valid for move charge
4278 * @vma: the vma the pte to be checked belongs
4279 * @addr: the address corresponding to the pte to be checked
4280 * @ptent: the pte to be checked
Daisuke Nishimura02491442010-03-10 15:22:17 -08004281 * @target: the pointer the target page or swap ent will be stored(can be NULL)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004282 *
4283 * Returns
4284 * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
4285 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4286 * move charge. if @target is not NULL, the page is stored in target->page
4287 * with extra refcnt got(Callers should handle it).
Daisuke Nishimura02491442010-03-10 15:22:17 -08004288 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4289 * target for charge migration. if @target is not NULL, the entry is stored
4290 * in target->ent.
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004291 *
4292 * Called with pte lock held.
4293 */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004294union mc_target {
4295 struct page *page;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004296 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004297};
4298
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004299enum mc_target_type {
4300 MC_TARGET_NONE, /* not used */
4301 MC_TARGET_PAGE,
Daisuke Nishimura02491442010-03-10 15:22:17 -08004302 MC_TARGET_SWAP,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004303};
4304
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004305static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4306 unsigned long addr, pte_t ptent)
4307{
4308 struct page *page = vm_normal_page(vma, addr, ptent);
4309
4310 if (!page || !page_mapped(page))
4311 return NULL;
4312 if (PageAnon(page)) {
4313 /* we don't move shared anon */
4314 if (!move_anon() || page_mapcount(page) > 2)
4315 return NULL;
Daisuke Nishimura87946a72010-05-26 14:42:39 -07004316 } else if (!move_file())
4317 /* we ignore mapcount for file pages */
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004318 return NULL;
4319 if (!get_page_unless_zero(page))
4320 return NULL;
4321
4322 return page;
4323}
4324
4325static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4326 unsigned long addr, pte_t ptent, swp_entry_t *entry)
4327{
4328 int usage_count;
4329 struct page *page = NULL;
4330 swp_entry_t ent = pte_to_swp_entry(ptent);
4331
4332 if (!move_anon() || non_swap_entry(ent))
4333 return NULL;
4334 usage_count = mem_cgroup_count_swap_user(ent, &page);
4335 if (usage_count > 1) { /* we don't move shared anon */
4336 if (page)
4337 put_page(page);
4338 return NULL;
4339 }
4340 if (do_swap_account)
4341 entry->val = ent.val;
4342
4343 return page;
4344}
4345
Daisuke Nishimura87946a72010-05-26 14:42:39 -07004346static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4347 unsigned long addr, pte_t ptent, swp_entry_t *entry)
4348{
4349 struct page *page = NULL;
4350 struct inode *inode;
4351 struct address_space *mapping;
4352 pgoff_t pgoff;
4353
4354 if (!vma->vm_file) /* anonymous vma */
4355 return NULL;
4356 if (!move_file())
4357 return NULL;
4358
4359 inode = vma->vm_file->f_path.dentry->d_inode;
4360 mapping = vma->vm_file->f_mapping;
4361 if (pte_none(ptent))
4362 pgoff = linear_page_index(vma, addr);
4363 else /* pte_file(ptent) is true */
4364 pgoff = pte_to_pgoff(ptent);
4365
4366 /* page is moved even if it's not RSS of this task(page-faulted). */
4367 if (!mapping_cap_swap_backed(mapping)) { /* normal file */
4368 page = find_get_page(mapping, pgoff);
4369 } else { /* shmem/tmpfs file. we should take account of swap too. */
4370 swp_entry_t ent;
4371 mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent);
4372 if (do_swap_account)
4373 entry->val = ent.val;
4374 }
4375
4376 return page;
4377}
4378
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004379static int is_target_pte_for_mc(struct vm_area_struct *vma,
4380 unsigned long addr, pte_t ptent, union mc_target *target)
4381{
Daisuke Nishimura02491442010-03-10 15:22:17 -08004382 struct page *page = NULL;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004383 struct page_cgroup *pc;
4384 int ret = 0;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004385 swp_entry_t ent = { .val = 0 };
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004386
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004387 if (pte_present(ptent))
4388 page = mc_handle_present_pte(vma, addr, ptent);
4389 else if (is_swap_pte(ptent))
4390 page = mc_handle_swap_pte(vma, addr, ptent, &ent);
Daisuke Nishimura87946a72010-05-26 14:42:39 -07004391 else if (pte_none(ptent) || pte_file(ptent))
4392 page = mc_handle_file_pte(vma, addr, ptent, &ent);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004393
4394 if (!page && !ent.val)
Daisuke Nishimura02491442010-03-10 15:22:17 -08004395 return 0;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004396 if (page) {
4397 pc = lookup_page_cgroup(page);
4398 /*
4399 * Do only loose check w/o page_cgroup lock.
4400 * mem_cgroup_move_account() checks the pc is valid or not under
4401 * the lock.
4402 */
4403 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
4404 ret = MC_TARGET_PAGE;
4405 if (target)
4406 target->page = page;
4407 }
4408 if (!ret || !target)
4409 put_page(page);
4410 }
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004411 /* There is a swap entry and a page doesn't exist or isn't charged */
4412 if (ent.val && !ret &&
KAMEZAWA Hiroyuki7f0f1542010-05-11 14:06:58 -07004413 css_id(&mc.from->css) == lookup_swap_cgroup(ent)) {
4414 ret = MC_TARGET_SWAP;
4415 if (target)
4416 target->ent = ent;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004417 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004418 return ret;
4419}
4420
4421static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4422 unsigned long addr, unsigned long end,
4423 struct mm_walk *walk)
4424{
4425 struct vm_area_struct *vma = walk->private;
4426 pte_t *pte;
4427 spinlock_t *ptl;
4428
4429 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4430 for (; addr != end; pte++, addr += PAGE_SIZE)
4431 if (is_target_pte_for_mc(vma, addr, *pte, NULL))
4432 mc.precharge++; /* increment precharge temporarily */
4433 pte_unmap_unlock(pte - 1, ptl);
4434 cond_resched();
4435
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004436 return 0;
4437}
4438
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004439static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4440{
4441 unsigned long precharge;
4442 struct vm_area_struct *vma;
4443
4444 down_read(&mm->mmap_sem);
4445 for (vma = mm->mmap; vma; vma = vma->vm_next) {
4446 struct mm_walk mem_cgroup_count_precharge_walk = {
4447 .pmd_entry = mem_cgroup_count_precharge_pte_range,
4448 .mm = mm,
4449 .private = vma,
4450 };
4451 if (is_vm_hugetlb_page(vma))
4452 continue;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004453 walk_page_range(vma->vm_start, vma->vm_end,
4454 &mem_cgroup_count_precharge_walk);
4455 }
4456 up_read(&mm->mmap_sem);
4457
4458 precharge = mc.precharge;
4459 mc.precharge = 0;
4460
4461 return precharge;
4462}
4463
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004464static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4465{
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004466 return mem_cgroup_do_precharge(mem_cgroup_count_precharge(mm));
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004467}
4468
4469static void mem_cgroup_clear_mc(void)
4470{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004471 struct mem_cgroup *from = mc.from;
4472 struct mem_cgroup *to = mc.to;
4473
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004474 /* we must uncharge all the leftover precharges from mc.to */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004475 if (mc.precharge) {
4476 __mem_cgroup_cancel_charge(mc.to, mc.precharge);
4477 mc.precharge = 0;
4478 }
4479 /*
4480 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4481 * we must uncharge here.
4482 */
4483 if (mc.moved_charge) {
4484 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
4485 mc.moved_charge = 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004486 }
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004487 /* we must fixup refcnts and charges */
4488 if (mc.moved_swap) {
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004489 /* uncharge swap account from the old cgroup */
4490 if (!mem_cgroup_is_root(mc.from))
4491 res_counter_uncharge(&mc.from->memsw,
4492 PAGE_SIZE * mc.moved_swap);
4493 __mem_cgroup_put(mc.from, mc.moved_swap);
4494
4495 if (!mem_cgroup_is_root(mc.to)) {
4496 /*
4497 * we charged both to->res and to->memsw, so we should
4498 * uncharge to->res.
4499 */
4500 res_counter_uncharge(&mc.to->res,
4501 PAGE_SIZE * mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004502 }
4503 /* we've already done mem_cgroup_get(mc.to) */
4504
4505 mc.moved_swap = 0;
4506 }
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004507 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004508 mc.from = NULL;
4509 mc.to = NULL;
Daisuke Nishimura8033b972010-03-10 15:22:16 -08004510 mc.moving_task = NULL;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004511 spin_unlock(&mc.lock);
4512 memcg_oom_recover(from);
4513 memcg_oom_recover(to);
Daisuke Nishimura8033b972010-03-10 15:22:16 -08004514 wake_up_all(&mc.waitq);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004515}
4516
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004517static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
4518 struct cgroup *cgroup,
4519 struct task_struct *p,
4520 bool threadgroup)
4521{
4522 int ret = 0;
4523 struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup);
4524
4525 if (mem->move_charge_at_immigrate) {
4526 struct mm_struct *mm;
4527 struct mem_cgroup *from = mem_cgroup_from_task(p);
4528
4529 VM_BUG_ON(from == mem);
4530
4531 mm = get_task_mm(p);
4532 if (!mm)
4533 return 0;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004534 /* We move charges only when we move a owner of the mm */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004535 if (mm->owner == p) {
4536 VM_BUG_ON(mc.from);
4537 VM_BUG_ON(mc.to);
4538 VM_BUG_ON(mc.precharge);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004539 VM_BUG_ON(mc.moved_charge);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004540 VM_BUG_ON(mc.moved_swap);
Daisuke Nishimura8033b972010-03-10 15:22:16 -08004541 VM_BUG_ON(mc.moving_task);
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004542 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004543 mc.from = from;
4544 mc.to = mem;
4545 mc.precharge = 0;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004546 mc.moved_charge = 0;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004547 mc.moved_swap = 0;
Daisuke Nishimura8033b972010-03-10 15:22:16 -08004548 mc.moving_task = current;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004549 spin_unlock(&mc.lock);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004550
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004551 ret = mem_cgroup_precharge_mc(mm);
4552 if (ret)
4553 mem_cgroup_clear_mc();
4554 }
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004555 mmput(mm);
4556 }
4557 return ret;
4558}
4559
4560static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
4561 struct cgroup *cgroup,
4562 struct task_struct *p,
4563 bool threadgroup)
4564{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004565 mem_cgroup_clear_mc();
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004566}
4567
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004568static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4569 unsigned long addr, unsigned long end,
4570 struct mm_walk *walk)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004571{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004572 int ret = 0;
4573 struct vm_area_struct *vma = walk->private;
4574 pte_t *pte;
4575 spinlock_t *ptl;
4576
4577retry:
4578 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4579 for (; addr != end; addr += PAGE_SIZE) {
4580 pte_t ptent = *(pte++);
4581 union mc_target target;
4582 int type;
4583 struct page *page;
4584 struct page_cgroup *pc;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004585 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004586
4587 if (!mc.precharge)
4588 break;
4589
4590 type = is_target_pte_for_mc(vma, addr, ptent, &target);
4591 switch (type) {
4592 case MC_TARGET_PAGE:
4593 page = target.page;
4594 if (isolate_lru_page(page))
4595 goto put;
4596 pc = lookup_page_cgroup(page);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004597 if (!mem_cgroup_move_account(pc,
4598 mc.from, mc.to, false)) {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004599 mc.precharge--;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004600 /* we uncharge from mc.from later. */
4601 mc.moved_charge++;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004602 }
4603 putback_lru_page(page);
4604put: /* is_target_pte_for_mc() gets the page */
4605 put_page(page);
4606 break;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004607 case MC_TARGET_SWAP:
4608 ent = target.ent;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004609 if (!mem_cgroup_move_swap_account(ent,
4610 mc.from, mc.to, false)) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08004611 mc.precharge--;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004612 /* we fixup refcnts and charges later. */
4613 mc.moved_swap++;
4614 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08004615 break;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004616 default:
4617 break;
4618 }
4619 }
4620 pte_unmap_unlock(pte - 1, ptl);
4621 cond_resched();
4622
4623 if (addr != end) {
4624 /*
4625 * We have consumed all precharges we got in can_attach().
4626 * We try charge one by one, but don't do any additional
4627 * charges to mc.to if we have failed in charge once in attach()
4628 * phase.
4629 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004630 ret = mem_cgroup_do_precharge(1);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004631 if (!ret)
4632 goto retry;
4633 }
4634
4635 return ret;
4636}
4637
4638static void mem_cgroup_move_charge(struct mm_struct *mm)
4639{
4640 struct vm_area_struct *vma;
4641
4642 lru_add_drain_all();
4643 down_read(&mm->mmap_sem);
4644 for (vma = mm->mmap; vma; vma = vma->vm_next) {
4645 int ret;
4646 struct mm_walk mem_cgroup_move_charge_walk = {
4647 .pmd_entry = mem_cgroup_move_charge_pte_range,
4648 .mm = mm,
4649 .private = vma,
4650 };
4651 if (is_vm_hugetlb_page(vma))
4652 continue;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004653 ret = walk_page_range(vma->vm_start, vma->vm_end,
4654 &mem_cgroup_move_charge_walk);
4655 if (ret)
4656 /*
4657 * means we have consumed all precharges and failed in
4658 * doing additional charge. Just abandon here.
4659 */
4660 break;
4661 }
4662 up_read(&mm->mmap_sem);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004663}
4664
Balbir Singh67e465a2008-02-07 00:13:54 -08004665static void mem_cgroup_move_task(struct cgroup_subsys *ss,
4666 struct cgroup *cont,
4667 struct cgroup *old_cont,
Ben Blumbe367d02009-09-23 15:56:31 -07004668 struct task_struct *p,
4669 bool threadgroup)
Balbir Singh67e465a2008-02-07 00:13:54 -08004670{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004671 struct mm_struct *mm;
4672
4673 if (!mc.to)
4674 /* no need to move charge */
4675 return;
4676
4677 mm = get_task_mm(p);
4678 if (mm) {
4679 mem_cgroup_move_charge(mm);
4680 mmput(mm);
4681 }
4682 mem_cgroup_clear_mc();
Balbir Singh67e465a2008-02-07 00:13:54 -08004683}
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07004684#else /* !CONFIG_MMU */
4685static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
4686 struct cgroup *cgroup,
4687 struct task_struct *p,
4688 bool threadgroup)
4689{
4690 return 0;
4691}
4692static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
4693 struct cgroup *cgroup,
4694 struct task_struct *p,
4695 bool threadgroup)
4696{
4697}
4698static void mem_cgroup_move_task(struct cgroup_subsys *ss,
4699 struct cgroup *cont,
4700 struct cgroup *old_cont,
4701 struct task_struct *p,
4702 bool threadgroup)
4703{
4704}
4705#endif
Balbir Singh67e465a2008-02-07 00:13:54 -08004706
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004707struct cgroup_subsys mem_cgroup_subsys = {
4708 .name = "memory",
4709 .subsys_id = mem_cgroup_subsys_id,
4710 .create = mem_cgroup_create,
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08004711 .pre_destroy = mem_cgroup_pre_destroy,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004712 .destroy = mem_cgroup_destroy,
4713 .populate = mem_cgroup_populate,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004714 .can_attach = mem_cgroup_can_attach,
4715 .cancel_attach = mem_cgroup_cancel_attach,
Balbir Singh67e465a2008-02-07 00:13:54 -08004716 .attach = mem_cgroup_move_task,
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004717 .early_init = 0,
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004718 .use_id = 1,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004719};
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004720
4721#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4722
4723static int __init disable_swap_account(char *s)
4724{
4725 really_do_swap_account = 0;
4726 return 1;
4727}
4728__setup("noswapaccount", disable_swap_account);
4729#endif