blob: 7a94ef6b35e2bc6c7cb48aeafc8206ece53b62a9 [file] [log] [blame]
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001/* memcontrol.c - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08006 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08009 * Memory thresholds
10 * Copyright (C) 2009 Nokia Corporation
11 * Author: Kirill A. Shutemov
12 *
Balbir Singh8cdea7c2008-02-07 00:13:50 -080013 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 */
23
24#include <linux/res_counter.h>
25#include <linux/memcontrol.h>
26#include <linux/cgroup.h>
Pavel Emelianov78fb7462008-02-07 00:13:51 -080027#include <linux/mm.h>
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -080028#include <linux/hugetlb.h>
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -080029#include <linux/pagemap.h>
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080030#include <linux/smp.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080031#include <linux/page-flags.h>
Balbir Singh66e17072008-02-07 00:13:56 -080032#include <linux/backing-dev.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080033#include <linux/bit_spinlock.h>
34#include <linux/rcupdate.h>
Balbir Singhe2224322009-04-02 16:57:39 -070035#include <linux/limits.h>
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -080036#include <linux/mutex.h>
Balbir Singhf64c3f52009-09-23 15:56:37 -070037#include <linux/rbtree.h>
Balbir Singhb6ac57d2008-04-29 01:00:19 -070038#include <linux/slab.h>
Balbir Singh66e17072008-02-07 00:13:56 -080039#include <linux/swap.h>
Daisuke Nishimura02491442010-03-10 15:22:17 -080040#include <linux/swapops.h>
Balbir Singh66e17072008-02-07 00:13:56 -080041#include <linux/spinlock.h>
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080042#include <linux/eventfd.h>
43#include <linux/sort.h>
Balbir Singh66e17072008-02-07 00:13:56 -080044#include <linux/fs.h>
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -080045#include <linux/seq_file.h>
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -070046#include <linux/vmalloc.h>
Christoph Lameterb69408e2008-10-18 20:26:14 -070047#include <linux/mm_inline.h>
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -070048#include <linux/page_cgroup.h>
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -080049#include <linux/cpu.h>
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -070050#include <linux/oom.h>
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -080051#include "internal.h"
Balbir Singh8cdea7c2008-02-07 00:13:50 -080052
Balbir Singh8697d332008-02-07 00:13:59 -080053#include <asm/uaccess.h>
54
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -070055#include <trace/events/vmscan.h>
56
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070057struct cgroup_subsys mem_cgroup_subsys __read_mostly;
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070058#define MEM_CGROUP_RECLAIM_RETRIES 5
Balbir Singh4b3bde42009-09-23 15:56:32 -070059struct mem_cgroup *root_mem_cgroup __read_mostly;
Balbir Singh8cdea7c2008-02-07 00:13:50 -080060
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080061#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
Li Zefan338c8432009-06-17 16:27:15 -070062/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080063int do_swap_account __read_mostly;
Michal Hockoa42c3902010-11-24 12:57:08 -080064
65/* for remember boot option*/
66#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED
67static int really_do_swap_account __initdata = 1;
68#else
69static int really_do_swap_account __initdata = 0;
70#endif
71
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080072#else
73#define do_swap_account (0)
74#endif
75
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -080076/*
77 * Per memcg event counter is incremented at every pagein/pageout. This counter
78 * is used for trigger some periodic events. This is straightforward and better
79 * than using jiffies etc. to handle periodic memcg event.
80 *
81 * These values will be used as !((event) & ((1 <<(thresh)) - 1))
82 */
83#define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */
84#define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080085
Balbir Singh8cdea7c2008-02-07 00:13:50 -080086/*
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080087 * Statistics for memory cgroup.
88 */
89enum mem_cgroup_stat_index {
90 /*
91 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
92 */
93 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
Balbir Singhd69b0422009-06-17 16:26:34 -070094 MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -080095 MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
Balaji Rao55e462b2008-05-01 04:35:12 -070096 MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */
97 MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */
Balbir Singh0c3e73e2009-09-23 15:56:42 -070098 MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -070099 MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */
100 /* incremented at every pagein/pageout */
101 MEM_CGROUP_EVENTS = MEM_CGROUP_STAT_DATA,
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -0700102 MEM_CGROUP_ON_MOVE, /* someone is moving account between groups */
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800103
104 MEM_CGROUP_STAT_NSTATS,
105};
106
107struct mem_cgroup_stat_cpu {
108 s64 count[MEM_CGROUP_STAT_NSTATS];
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800109};
110
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800111/*
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800112 * per-zone information in memory controller.
113 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800114struct mem_cgroup_per_zone {
KAMEZAWA Hiroyuki072c56c12008-02-07 00:14:39 -0800115 /*
116 * spin_lock to protect the per cgroup LRU
117 */
Christoph Lameterb69408e2008-10-18 20:26:14 -0700118 struct list_head lists[NR_LRU_LISTS];
119 unsigned long count[NR_LRU_LISTS];
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -0800120
121 struct zone_reclaim_stat reclaim_stat;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700122 struct rb_node tree_node; /* RB tree node */
123 unsigned long long usage_in_excess;/* Set to the value by which */
124 /* the soft limit is exceeded*/
125 bool on_tree;
Balbir Singh4e416952009-09-23 15:56:39 -0700126 struct mem_cgroup *mem; /* Back pointer, we cannot */
127 /* use container_of */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800128};
129/* Macro for accessing counter */
130#define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)])
131
132struct mem_cgroup_per_node {
133 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
134};
135
136struct mem_cgroup_lru_info {
137 struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
138};
139
140/*
Balbir Singhf64c3f52009-09-23 15:56:37 -0700141 * Cgroups above their limits are maintained in a RB-Tree, independent of
142 * their hierarchy representation
143 */
144
145struct mem_cgroup_tree_per_zone {
146 struct rb_root rb_root;
147 spinlock_t lock;
148};
149
150struct mem_cgroup_tree_per_node {
151 struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
152};
153
154struct mem_cgroup_tree {
155 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
156};
157
158static struct mem_cgroup_tree soft_limit_tree __read_mostly;
159
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800160struct mem_cgroup_threshold {
161 struct eventfd_ctx *eventfd;
162 u64 threshold;
163};
164
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700165/* For threshold */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800166struct mem_cgroup_threshold_ary {
167 /* An array index points to threshold just below usage. */
Phil Carmody5407a562010-05-26 14:42:42 -0700168 int current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800169 /* Size of entries[] */
170 unsigned int size;
171 /* Array of thresholds */
172 struct mem_cgroup_threshold entries[0];
173};
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700174
175struct mem_cgroup_thresholds {
176 /* Primary thresholds array */
177 struct mem_cgroup_threshold_ary *primary;
178 /*
179 * Spare threshold array.
180 * This is needed to make mem_cgroup_unregister_event() "never fail".
181 * It must be able to store at least primary->size - 1 entries.
182 */
183 struct mem_cgroup_threshold_ary *spare;
184};
185
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700186/* for OOM */
187struct mem_cgroup_eventfd_list {
188 struct list_head list;
189 struct eventfd_ctx *eventfd;
190};
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800191
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800192static void mem_cgroup_threshold(struct mem_cgroup *mem);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700193static void mem_cgroup_oom_notify(struct mem_cgroup *mem);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800194
Balbir Singhf64c3f52009-09-23 15:56:37 -0700195/*
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800196 * The memory controller data structure. The memory controller controls both
197 * page cache and RSS per cgroup. We would eventually like to provide
198 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
199 * to help the administrator determine what knobs to tune.
200 *
201 * TODO: Add a water mark for the memory controller. Reclaim will begin when
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800202 * we hit the water mark. May be even add a low water mark, such that
203 * no reclaim occurs from a cgroup at it's low water mark, this is
204 * a feature that will be implemented much later in the future.
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800205 */
206struct mem_cgroup {
207 struct cgroup_subsys_state css;
208 /*
209 * the counter to account for memory usage
210 */
211 struct res_counter res;
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800212 /*
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800213 * the counter to account for mem+swap usage.
214 */
215 struct res_counter memsw;
216 /*
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800217 * Per cgroup active and inactive list, similar to the
218 * per zone LRU lists.
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800219 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800220 struct mem_cgroup_lru_info info;
KAMEZAWA Hiroyuki072c56c12008-02-07 00:14:39 -0800221
KOSAKI Motohiro2733c062009-01-07 18:08:23 -0800222 /*
223 protect against reclaim related member.
224 */
225 spinlock_t reclaim_param_lock;
226
Balbir Singh6d61ef42009-01-07 18:08:06 -0800227 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200228 * While reclaiming in a hierarchy, we cache the last child we
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -0700229 * reclaimed from.
Balbir Singh6d61ef42009-01-07 18:08:06 -0800230 */
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -0700231 int last_scanned_child;
Balbir Singh18f59ea2009-01-07 18:08:07 -0800232 /*
233 * Should the accounting and control be hierarchical, per subtree?
234 */
235 bool use_hierarchy;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -0800236 atomic_t oom_lock;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800237 atomic_t refcnt;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800238
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -0800239 unsigned int swappiness;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -0700240 /* OOM-Killer disable */
241 int oom_kill_disable;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -0800242
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -0700243 /* set when res.limit == memsw.limit */
244 bool memsw_is_minimum;
245
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800246 /* protect arrays of thresholds */
247 struct mutex thresholds_lock;
248
249 /* thresholds for memory usage. RCU-protected */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700250 struct mem_cgroup_thresholds thresholds;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -0700251
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800252 /* thresholds for mem+swap usage. RCU-protected */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700253 struct mem_cgroup_thresholds memsw_thresholds;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -0700254
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700255 /* For oom notifier event fd */
256 struct list_head oom_notify;
257
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800258 /*
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800259 * Should we move charges of a task when a task is moved into this
260 * mem_cgroup ? And what type of charges should we move ?
261 */
262 unsigned long move_charge_at_immigrate;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800263 /*
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800264 * percpu counter.
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800265 */
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800266 struct mem_cgroup_stat_cpu *stat;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700267 /*
268 * used when a cpu is offlined or other synchronizations
269 * See mem_cgroup_read_stat().
270 */
271 struct mem_cgroup_stat_cpu nocpu_base;
272 spinlock_t pcp_counter_lock;
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800273};
274
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800275/* Stuffs for move charges at task migration. */
276/*
277 * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
278 * left-shifted bitmap of these types.
279 */
280enum move_type {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800281 MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700282 MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800283 NR_MOVE_TYPE,
284};
285
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800286/* "mc" and its members are protected by cgroup_mutex */
287static struct move_charge_struct {
Daisuke Nishimurab1dd6932010-11-24 12:57:06 -0800288 spinlock_t lock; /* for from, to */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800289 struct mem_cgroup *from;
290 struct mem_cgroup *to;
291 unsigned long precharge;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -0800292 unsigned long moved_charge;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -0800293 unsigned long moved_swap;
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800294 struct task_struct *moving_task; /* a task moving charges */
295 wait_queue_head_t waitq; /* a waitq for other context */
296} mc = {
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -0700297 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800298 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
299};
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800300
Daisuke Nishimura90254a62010-05-26 14:42:38 -0700301static bool move_anon(void)
302{
303 return test_bit(MOVE_CHARGE_TYPE_ANON,
304 &mc.to->move_charge_at_immigrate);
305}
306
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700307static bool move_file(void)
308{
309 return test_bit(MOVE_CHARGE_TYPE_FILE,
310 &mc.to->move_charge_at_immigrate);
311}
312
Balbir Singh4e416952009-09-23 15:56:39 -0700313/*
314 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
315 * limit reclaim to prevent infinite loops, if they ever occur.
316 */
317#define MEM_CGROUP_MAX_RECLAIM_LOOPS (100)
318#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2)
319
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800320enum charge_type {
321 MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
322 MEM_CGROUP_CHARGE_TYPE_MAPPED,
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700323 MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700324 MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -0800325 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
KAMEZAWA Hiroyuki8a9478ca2009-06-17 16:27:17 -0700326 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700327 NR_CHARGE_TYPE,
328};
329
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700330/* only for here (for easy reading.) */
331#define PCGF_CACHE (1UL << PCG_CACHE)
332#define PCGF_USED (1UL << PCG_USED)
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700333#define PCGF_LOCK (1UL << PCG_LOCK)
Balbir Singh4b3bde42009-09-23 15:56:32 -0700334/* Not used, but added here for completeness */
335#define PCGF_ACCT (1UL << PCG_ACCT)
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800336
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800337/* for encoding cft->private value on file */
338#define _MEM (0)
339#define _MEMSWAP (1)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700340#define _OOM_TYPE (2)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800341#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
342#define MEMFILE_TYPE(val) (((val) >> 16) & 0xffff)
343#define MEMFILE_ATTR(val) ((val) & 0xffff)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700344/* Used for OOM nofiier */
345#define OOM_CONTROL (0)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800346
Balbir Singh75822b42009-09-23 15:56:38 -0700347/*
348 * Reclaim flags for mem_cgroup_hierarchical_reclaim
349 */
350#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0
351#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
352#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1
353#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
Balbir Singh4e416952009-09-23 15:56:39 -0700354#define MEM_CGROUP_RECLAIM_SOFT_BIT 0x2
355#define MEM_CGROUP_RECLAIM_SOFT (1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
Balbir Singh75822b42009-09-23 15:56:38 -0700356
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800357static void mem_cgroup_get(struct mem_cgroup *mem);
358static void mem_cgroup_put(struct mem_cgroup *mem);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -0800359static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -0800360static void drain_all_stock_async(void);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800361
Balbir Singhf64c3f52009-09-23 15:56:37 -0700362static struct mem_cgroup_per_zone *
363mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
364{
365 return &mem->info.nodeinfo[nid]->zoneinfo[zid];
366}
367
Wu Fengguangd3242362009-12-16 12:19:59 +0100368struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
369{
370 return &mem->css;
371}
372
Balbir Singhf64c3f52009-09-23 15:56:37 -0700373static struct mem_cgroup_per_zone *
374page_cgroup_zoneinfo(struct page_cgroup *pc)
375{
376 struct mem_cgroup *mem = pc->mem_cgroup;
377 int nid = page_cgroup_nid(pc);
378 int zid = page_cgroup_zid(pc);
379
380 if (!mem)
381 return NULL;
382
383 return mem_cgroup_zoneinfo(mem, nid, zid);
384}
385
386static struct mem_cgroup_tree_per_zone *
387soft_limit_tree_node_zone(int nid, int zid)
388{
389 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
390}
391
392static struct mem_cgroup_tree_per_zone *
393soft_limit_tree_from_page(struct page *page)
394{
395 int nid = page_to_nid(page);
396 int zid = page_zonenum(page);
397
398 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
399}
400
401static void
Balbir Singh4e416952009-09-23 15:56:39 -0700402__mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
Balbir Singhf64c3f52009-09-23 15:56:37 -0700403 struct mem_cgroup_per_zone *mz,
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700404 struct mem_cgroup_tree_per_zone *mctz,
405 unsigned long long new_usage_in_excess)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700406{
407 struct rb_node **p = &mctz->rb_root.rb_node;
408 struct rb_node *parent = NULL;
409 struct mem_cgroup_per_zone *mz_node;
410
411 if (mz->on_tree)
412 return;
413
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700414 mz->usage_in_excess = new_usage_in_excess;
415 if (!mz->usage_in_excess)
416 return;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700417 while (*p) {
418 parent = *p;
419 mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
420 tree_node);
421 if (mz->usage_in_excess < mz_node->usage_in_excess)
422 p = &(*p)->rb_left;
423 /*
424 * We can't avoid mem cgroups that are over their soft
425 * limit by the same amount
426 */
427 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
428 p = &(*p)->rb_right;
429 }
430 rb_link_node(&mz->tree_node, parent, p);
431 rb_insert_color(&mz->tree_node, &mctz->rb_root);
432 mz->on_tree = true;
Balbir Singh4e416952009-09-23 15:56:39 -0700433}
434
435static void
436__mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
437 struct mem_cgroup_per_zone *mz,
438 struct mem_cgroup_tree_per_zone *mctz)
439{
440 if (!mz->on_tree)
441 return;
442 rb_erase(&mz->tree_node, &mctz->rb_root);
443 mz->on_tree = false;
444}
445
446static void
Balbir Singhf64c3f52009-09-23 15:56:37 -0700447mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
448 struct mem_cgroup_per_zone *mz,
449 struct mem_cgroup_tree_per_zone *mctz)
450{
451 spin_lock(&mctz->lock);
Balbir Singh4e416952009-09-23 15:56:39 -0700452 __mem_cgroup_remove_exceeded(mem, mz, mctz);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700453 spin_unlock(&mctz->lock);
454}
455
Balbir Singhf64c3f52009-09-23 15:56:37 -0700456
457static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
458{
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700459 unsigned long long excess;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700460 struct mem_cgroup_per_zone *mz;
461 struct mem_cgroup_tree_per_zone *mctz;
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700462 int nid = page_to_nid(page);
463 int zid = page_zonenum(page);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700464 mctz = soft_limit_tree_from_page(page);
465
466 /*
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700467 * Necessary to update all ancestors when hierarchy is used.
468 * because their event counter is not touched.
Balbir Singhf64c3f52009-09-23 15:56:37 -0700469 */
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700470 for (; mem; mem = parent_mem_cgroup(mem)) {
471 mz = mem_cgroup_zoneinfo(mem, nid, zid);
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700472 excess = res_counter_soft_limit_excess(&mem->res);
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700473 /*
474 * We have to update the tree if mz is on RB-tree or
475 * mem is over its softlimit.
476 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700477 if (excess || mz->on_tree) {
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700478 spin_lock(&mctz->lock);
479 /* if on-tree, remove it */
480 if (mz->on_tree)
481 __mem_cgroup_remove_exceeded(mem, mz, mctz);
482 /*
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700483 * Insert again. mz->usage_in_excess will be updated.
484 * If excess is 0, no tree ops.
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700485 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700486 __mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700487 spin_unlock(&mctz->lock);
488 }
Balbir Singhf64c3f52009-09-23 15:56:37 -0700489 }
490}
491
492static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
493{
494 int node, zone;
495 struct mem_cgroup_per_zone *mz;
496 struct mem_cgroup_tree_per_zone *mctz;
497
498 for_each_node_state(node, N_POSSIBLE) {
499 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
500 mz = mem_cgroup_zoneinfo(mem, node, zone);
501 mctz = soft_limit_tree_node_zone(node, zone);
502 mem_cgroup_remove_exceeded(mem, mz, mctz);
503 }
504 }
505}
506
Balbir Singh4e416952009-09-23 15:56:39 -0700507static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem)
508{
509 return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT;
510}
511
512static struct mem_cgroup_per_zone *
513__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
514{
515 struct rb_node *rightmost = NULL;
KAMEZAWA Hiroyuki26251ea2009-10-01 15:44:08 -0700516 struct mem_cgroup_per_zone *mz;
Balbir Singh4e416952009-09-23 15:56:39 -0700517
518retry:
KAMEZAWA Hiroyuki26251ea2009-10-01 15:44:08 -0700519 mz = NULL;
Balbir Singh4e416952009-09-23 15:56:39 -0700520 rightmost = rb_last(&mctz->rb_root);
521 if (!rightmost)
522 goto done; /* Nothing to reclaim from */
523
524 mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
525 /*
526 * Remove the node now but someone else can add it back,
527 * we will to add it back at the end of reclaim to its correct
528 * position in the tree.
529 */
530 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
531 if (!res_counter_soft_limit_excess(&mz->mem->res) ||
532 !css_tryget(&mz->mem->css))
533 goto retry;
534done:
535 return mz;
536}
537
538static struct mem_cgroup_per_zone *
539mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
540{
541 struct mem_cgroup_per_zone *mz;
542
543 spin_lock(&mctz->lock);
544 mz = __mem_cgroup_largest_soft_limit_node(mctz);
545 spin_unlock(&mctz->lock);
546 return mz;
547}
548
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700549/*
550 * Implementation Note: reading percpu statistics for memcg.
551 *
552 * Both of vmstat[] and percpu_counter has threshold and do periodic
553 * synchronization to implement "quick" read. There are trade-off between
554 * reading cost and precision of value. Then, we may have a chance to implement
555 * a periodic synchronizion of counter in memcg's counter.
556 *
557 * But this _read() function is used for user interface now. The user accounts
558 * memory usage by memory cgroup and he _always_ requires exact value because
559 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
560 * have to visit all online cpus and make sum. So, for now, unnecessary
561 * synchronization is not implemented. (just implemented for cpu hotplug)
562 *
563 * If there are kernel internal actions which can make use of some not-exact
564 * value, and reading all cpu value can be performance bottleneck in some
565 * common workload, threashold and synchonization as vmstat[] should be
566 * implemented.
567 */
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800568static s64 mem_cgroup_read_stat(struct mem_cgroup *mem,
569 enum mem_cgroup_stat_index idx)
570{
571 int cpu;
572 s64 val = 0;
573
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700574 get_online_cpus();
575 for_each_online_cpu(cpu)
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800576 val += per_cpu(mem->stat->count[idx], cpu);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700577#ifdef CONFIG_HOTPLUG_CPU
578 spin_lock(&mem->pcp_counter_lock);
579 val += mem->nocpu_base.count[idx];
580 spin_unlock(&mem->pcp_counter_lock);
581#endif
582 put_online_cpus();
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800583 return val;
584}
585
586static s64 mem_cgroup_local_usage(struct mem_cgroup *mem)
587{
588 s64 ret;
589
590 ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
591 ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
592 return ret;
593}
594
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700595static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
596 bool charge)
597{
598 int val = (charge) ? 1 : -1;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800599 this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700600}
601
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700602static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800603 bool file, int nr_pages)
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800604{
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800605 preempt_disable();
606
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800607 if (file)
608 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], nr_pages);
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800609 else
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800610 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], nr_pages);
Balaji Rao55e462b2008-05-01 04:35:12 -0700611
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800612 /* pagein of a big page is an event. So, ignore page size */
613 if (nr_pages > 0)
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800614 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]);
Balaji Rao55e462b2008-05-01 04:35:12 -0700615 else
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800616 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]);
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800617
618 __this_cpu_add(mem->stat->count[MEM_CGROUP_EVENTS], nr_pages);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800619
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800620 preempt_enable();
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800621}
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800622
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700623static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
Christoph Lameterb69408e2008-10-18 20:26:14 -0700624 enum lru_list idx)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800625{
626 int nid, zid;
627 struct mem_cgroup_per_zone *mz;
628 u64 total = 0;
629
630 for_each_online_node(nid)
631 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
632 mz = mem_cgroup_zoneinfo(mem, nid, zid);
633 total += MEM_CGROUP_ZSTAT(mz, idx);
634 }
635 return total;
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800636}
637
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800638static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift)
639{
640 s64 val;
641
642 val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]);
643
644 return !(val & ((1 << event_mask_shift) - 1));
645}
646
647/*
648 * Check events in order.
649 *
650 */
651static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
652{
653 /* threshold event is triggered in finer grain than soft limit */
654 if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) {
655 mem_cgroup_threshold(mem);
656 if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH)))
657 mem_cgroup_update_tree(mem, page);
658 }
659}
660
Hugh Dickinsd5b69e32008-03-04 14:29:10 -0800661static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800662{
663 return container_of(cgroup_subsys_state(cont,
664 mem_cgroup_subsys_id), struct mem_cgroup,
665 css);
666}
667
Balbir Singhcf475ad2008-04-29 01:00:16 -0700668struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800669{
Balbir Singh31a78f22008-09-28 23:09:31 +0100670 /*
671 * mm_update_next_owner() may clear mm->owner to NULL
672 * if it races with swapoff, page migration, etc.
673 * So this can be called with p == NULL.
674 */
675 if (unlikely(!p))
676 return NULL;
677
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800678 return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
679 struct mem_cgroup, css);
680}
681
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800682static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
683{
684 struct mem_cgroup *mem = NULL;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700685
686 if (!mm)
687 return NULL;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800688 /*
689 * Because we have no locks, mm->owner's may be being moved to other
690 * cgroup. We use css_tryget() here even if this looks
691 * pessimistic (rather than adding locks here).
692 */
693 rcu_read_lock();
694 do {
695 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
696 if (unlikely(!mem))
697 break;
698 } while (!css_tryget(&mem->css));
699 rcu_read_unlock();
700 return mem;
701}
702
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700703/* The caller has to guarantee "mem" exists before calling this */
704static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700705{
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700706 struct cgroup_subsys_state *css;
707 int found;
708
709 if (!mem) /* ROOT cgroup has the smallest ID */
710 return root_mem_cgroup; /*css_put/get against root is ignored*/
711 if (!mem->use_hierarchy) {
712 if (css_tryget(&mem->css))
713 return mem;
714 return NULL;
715 }
716 rcu_read_lock();
717 /*
718 * searching a memory cgroup which has the smallest ID under given
719 * ROOT cgroup. (ID >= 1)
720 */
721 css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found);
722 if (css && css_tryget(css))
723 mem = container_of(css, struct mem_cgroup, css);
724 else
725 mem = NULL;
726 rcu_read_unlock();
727 return mem;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700728}
729
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700730static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter,
731 struct mem_cgroup *root,
732 bool cond)
733{
734 int nextid = css_id(&iter->css) + 1;
735 int found;
736 int hierarchy_used;
737 struct cgroup_subsys_state *css;
738
739 hierarchy_used = iter->use_hierarchy;
740
741 css_put(&iter->css);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700742 /* If no ROOT, walk all, ignore hierarchy */
743 if (!cond || (root && !hierarchy_used))
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700744 return NULL;
745
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700746 if (!root)
747 root = root_mem_cgroup;
748
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700749 do {
750 iter = NULL;
751 rcu_read_lock();
752
753 css = css_get_next(&mem_cgroup_subsys, nextid,
754 &root->css, &found);
755 if (css && css_tryget(css))
756 iter = container_of(css, struct mem_cgroup, css);
757 rcu_read_unlock();
758 /* If css is NULL, no more cgroups will be found */
759 nextid = found + 1;
760 } while (css && !iter);
761
762 return iter;
763}
764/*
765 * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please
766 * be careful that "break" loop is not allowed. We have reference count.
767 * Instead of that modify "cond" to be false and "continue" to exit the loop.
768 */
769#define for_each_mem_cgroup_tree_cond(iter, root, cond) \
770 for (iter = mem_cgroup_start_loop(root);\
771 iter != NULL;\
772 iter = mem_cgroup_get_next(iter, root, cond))
773
774#define for_each_mem_cgroup_tree(iter, root) \
775 for_each_mem_cgroup_tree_cond(iter, root, true)
776
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700777#define for_each_mem_cgroup_all(iter) \
778 for_each_mem_cgroup_tree_cond(iter, NULL, true)
779
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700780
Balbir Singh4b3bde42009-09-23 15:56:32 -0700781static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
782{
783 return (mem == root_mem_cgroup);
784}
785
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800786/*
787 * Following LRU functions are allowed to be used without PCG_LOCK.
788 * Operations are called by routine of global LRU independently from memcg.
789 * What we have to take care of here is validness of pc->mem_cgroup.
790 *
791 * Changes to pc->mem_cgroup happens when
792 * 1. charge
793 * 2. moving account
794 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
795 * It is added to LRU before charge.
796 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
797 * When moving account, the page is not on LRU. It's isolated.
798 */
799
800void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800801{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800802 struct page_cgroup *pc;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800803 struct mem_cgroup_per_zone *mz;
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700804
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800805 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800806 return;
807 pc = lookup_page_cgroup(page);
808 /* can happen while we handle swapcache. */
Balbir Singh4b3bde42009-09-23 15:56:32 -0700809 if (!TestClearPageCgroupAcctLRU(pc))
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800810 return;
Balbir Singh4b3bde42009-09-23 15:56:32 -0700811 VM_BUG_ON(!pc->mem_cgroup);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800812 /*
813 * We don't check PCG_USED bit. It's cleared when the "page" is finally
814 * removed from global LRU.
815 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800816 mz = page_cgroup_zoneinfo(pc);
KAMEZAWA Hiroyukiece35ca2011-01-20 14:44:24 -0800817 /* huge page split is done under lru_lock. so, we have no races. */
818 MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page);
Balbir Singh4b3bde42009-09-23 15:56:32 -0700819 if (mem_cgroup_is_root(pc->mem_cgroup))
820 return;
821 VM_BUG_ON(list_empty(&pc->lru));
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800822 list_del_init(&pc->lru);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800823}
824
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800825void mem_cgroup_del_lru(struct page *page)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800826{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800827 mem_cgroup_del_lru_list(page, page_lru(page));
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800828}
829
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800830void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
Balbir Singh66e17072008-02-07 00:13:56 -0800831{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800832 struct mem_cgroup_per_zone *mz;
833 struct page_cgroup *pc;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800834
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800835 if (mem_cgroup_disabled())
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700836 return;
Christoph Lameterb69408e2008-10-18 20:26:14 -0700837
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800838 pc = lookup_page_cgroup(page);
Daisuke Nishimurabd112db2009-01-15 13:51:11 -0800839 /*
840 * Used bit is set without atomic ops but after smp_wmb().
841 * For making pc->mem_cgroup visible, insert smp_rmb() here.
842 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800843 smp_rmb();
Balbir Singh4b3bde42009-09-23 15:56:32 -0700844 /* unused or root page is not rotated. */
845 if (!PageCgroupUsed(pc) || mem_cgroup_is_root(pc->mem_cgroup))
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800846 return;
847 mz = page_cgroup_zoneinfo(pc);
Christoph Lameterb69408e2008-10-18 20:26:14 -0700848 list_move(&pc->lru, &mz->lists[lru]);
Balbir Singh66e17072008-02-07 00:13:56 -0800849}
850
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800851void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
852{
853 struct page_cgroup *pc;
854 struct mem_cgroup_per_zone *mz;
855
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800856 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800857 return;
858 pc = lookup_page_cgroup(page);
Balbir Singh4b3bde42009-09-23 15:56:32 -0700859 VM_BUG_ON(PageCgroupAcctLRU(pc));
Daisuke Nishimurabd112db2009-01-15 13:51:11 -0800860 /*
861 * Used bit is set without atomic ops but after smp_wmb().
862 * For making pc->mem_cgroup visible, insert smp_rmb() here.
863 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800864 smp_rmb();
865 if (!PageCgroupUsed(pc))
866 return;
867
868 mz = page_cgroup_zoneinfo(pc);
KAMEZAWA Hiroyukiece35ca2011-01-20 14:44:24 -0800869 /* huge page split is done under lru_lock. so, we have no races. */
870 MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page);
Balbir Singh4b3bde42009-09-23 15:56:32 -0700871 SetPageCgroupAcctLRU(pc);
872 if (mem_cgroup_is_root(pc->mem_cgroup))
873 return;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800874 list_add(&pc->lru, &mz->lists[lru]);
875}
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800876
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800877/*
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800878 * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to
879 * lru because the page may.be reused after it's fully uncharged (because of
880 * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge
881 * it again. This function is only used to charge SwapCache. It's done under
882 * lock_page and expected that zone->lru_lock is never held.
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800883 */
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800884static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page)
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800885{
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800886 unsigned long flags;
887 struct zone *zone = page_zone(page);
888 struct page_cgroup *pc = lookup_page_cgroup(page);
889
890 spin_lock_irqsave(&zone->lru_lock, flags);
891 /*
892 * Forget old LRU when this page_cgroup is *not* used. This Used bit
893 * is guarded by lock_page() because the page is SwapCache.
894 */
895 if (!PageCgroupUsed(pc))
896 mem_cgroup_del_lru_list(page, page_lru(page));
897 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800898}
899
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800900static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
901{
902 unsigned long flags;
903 struct zone *zone = page_zone(page);
904 struct page_cgroup *pc = lookup_page_cgroup(page);
905
906 spin_lock_irqsave(&zone->lru_lock, flags);
907 /* link when the page is linked to LRU but page_cgroup isn't */
Balbir Singh4b3bde42009-09-23 15:56:32 -0700908 if (PageLRU(page) && !PageCgroupAcctLRU(pc))
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800909 mem_cgroup_add_lru_list(page, page_lru(page));
910 spin_unlock_irqrestore(&zone->lru_lock, flags);
911}
912
913
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800914void mem_cgroup_move_lists(struct page *page,
915 enum lru_list from, enum lru_list to)
916{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800917 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800918 return;
919 mem_cgroup_del_lru_list(page, from);
920 mem_cgroup_add_lru_list(page, to);
921}
922
David Rientjes4c4a2212008-02-07 00:14:06 -0800923int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
924{
925 int ret;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700926 struct mem_cgroup *curr = NULL;
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -0700927 struct task_struct *p;
David Rientjes4c4a2212008-02-07 00:14:06 -0800928
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -0700929 p = find_lock_task_mm(task);
930 if (!p)
931 return 0;
932 curr = try_get_mem_cgroup_from_mm(p->mm);
933 task_unlock(p);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700934 if (!curr)
935 return 0;
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -0800936 /*
937 * We should check use_hierarchy of "mem" not "curr". Because checking
938 * use_hierarchy of "curr" here make this function true if hierarchy is
939 * enabled in "curr" and "curr" is a child of "mem" in *cgroup*
940 * hierarchy(even if use_hierarchy is disabled in "mem").
941 */
942 if (mem->use_hierarchy)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700943 ret = css_is_ancestor(&curr->css, &mem->css);
944 else
945 ret = (curr == mem);
946 css_put(&curr->css);
David Rientjes4c4a2212008-02-07 00:14:06 -0800947 return ret;
948}
949
KOSAKI Motohiroc772be92009-01-07 18:08:25 -0800950static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800951{
952 unsigned long active;
953 unsigned long inactive;
KOSAKI Motohiroc772be92009-01-07 18:08:25 -0800954 unsigned long gb;
955 unsigned long inactive_ratio;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800956
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700957 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
958 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800959
KOSAKI Motohiroc772be92009-01-07 18:08:25 -0800960 gb = (inactive + active) >> (30 - PAGE_SHIFT);
961 if (gb)
962 inactive_ratio = int_sqrt(10 * gb);
963 else
964 inactive_ratio = 1;
965
966 if (present_pages) {
967 present_pages[0] = inactive;
968 present_pages[1] = active;
969 }
970
971 return inactive_ratio;
972}
973
974int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
975{
976 unsigned long active;
977 unsigned long inactive;
978 unsigned long present_pages[2];
979 unsigned long inactive_ratio;
980
981 inactive_ratio = calc_inactive_ratio(memcg, present_pages);
982
983 inactive = present_pages[0];
984 active = present_pages[1];
985
986 if (inactive * inactive_ratio < active)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800987 return 1;
988
989 return 0;
990}
991
Rik van Riel56e49d22009-06-16 15:32:28 -0700992int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
993{
994 unsigned long active;
995 unsigned long inactive;
996
997 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
998 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
999
1000 return (active > inactive);
1001}
1002
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -08001003unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
1004 struct zone *zone,
1005 enum lru_list lru)
1006{
KOSAKI Motohiro13d7e3a2010-08-10 18:03:06 -07001007 int nid = zone_to_nid(zone);
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -08001008 int zid = zone_idx(zone);
1009 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
1010
1011 return MEM_CGROUP_ZSTAT(mz, lru);
1012}
1013
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -08001014struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
1015 struct zone *zone)
1016{
KOSAKI Motohiro13d7e3a2010-08-10 18:03:06 -07001017 int nid = zone_to_nid(zone);
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -08001018 int zid = zone_idx(zone);
1019 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
1020
1021 return &mz->reclaim_stat;
1022}
1023
1024struct zone_reclaim_stat *
1025mem_cgroup_get_reclaim_stat_from_page(struct page *page)
1026{
1027 struct page_cgroup *pc;
1028 struct mem_cgroup_per_zone *mz;
1029
1030 if (mem_cgroup_disabled())
1031 return NULL;
1032
1033 pc = lookup_page_cgroup(page);
Daisuke Nishimurabd112db2009-01-15 13:51:11 -08001034 /*
1035 * Used bit is set without atomic ops but after smp_wmb().
1036 * For making pc->mem_cgroup visible, insert smp_rmb() here.
1037 */
1038 smp_rmb();
1039 if (!PageCgroupUsed(pc))
1040 return NULL;
1041
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -08001042 mz = page_cgroup_zoneinfo(pc);
1043 if (!mz)
1044 return NULL;
1045
1046 return &mz->reclaim_stat;
1047}
1048
Balbir Singh66e17072008-02-07 00:13:56 -08001049unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
1050 struct list_head *dst,
1051 unsigned long *scanned, int order,
1052 int mode, struct zone *z,
1053 struct mem_cgroup *mem_cont,
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001054 int active, int file)
Balbir Singh66e17072008-02-07 00:13:56 -08001055{
1056 unsigned long nr_taken = 0;
1057 struct page *page;
1058 unsigned long scan;
1059 LIST_HEAD(pc_list);
1060 struct list_head *src;
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001061 struct page_cgroup *pc, *tmp;
KOSAKI Motohiro13d7e3a2010-08-10 18:03:06 -07001062 int nid = zone_to_nid(z);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001063 int zid = zone_idx(z);
1064 struct mem_cgroup_per_zone *mz;
Johannes Weinerb7c46d12009-09-21 17:02:56 -07001065 int lru = LRU_FILE * file + active;
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001066 int ret;
Balbir Singh66e17072008-02-07 00:13:56 -08001067
Balbir Singhcf475ad2008-04-29 01:00:16 -07001068 BUG_ON(!mem_cont);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001069 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
Christoph Lameterb69408e2008-10-18 20:26:14 -07001070 src = &mz->lists[lru];
Balbir Singh66e17072008-02-07 00:13:56 -08001071
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001072 scan = 0;
1073 list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
Hugh Dickins436c65412008-02-07 00:14:12 -08001074 if (scan >= nr_to_scan)
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001075 break;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001076
1077 page = pc->page;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07001078 if (unlikely(!PageCgroupUsed(pc)))
1079 continue;
Hugh Dickins436c65412008-02-07 00:14:12 -08001080 if (unlikely(!PageLRU(page)))
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001081 continue;
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001082
Hugh Dickins436c65412008-02-07 00:14:12 -08001083 scan++;
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001084 ret = __isolate_lru_page(page, mode, file);
1085 switch (ret) {
1086 case 0:
Balbir Singh66e17072008-02-07 00:13:56 -08001087 list_move(&page->lru, dst);
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001088 mem_cgroup_del_lru(page);
Rik van Riel2c888cf2011-01-13 15:47:13 -08001089 nr_taken += hpage_nr_pages(page);
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001090 break;
1091 case -EBUSY:
1092 /* we don't affect global LRU but rotate in our LRU */
1093 mem_cgroup_rotate_lru_list(page, page_lru(page));
1094 break;
1095 default:
1096 break;
Balbir Singh66e17072008-02-07 00:13:56 -08001097 }
1098 }
1099
Balbir Singh66e17072008-02-07 00:13:56 -08001100 *scanned = scan;
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -07001101
1102 trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken,
1103 0, 0, 0, mode);
1104
Balbir Singh66e17072008-02-07 00:13:56 -08001105 return nr_taken;
1106}
1107
Balbir Singh6d61ef42009-01-07 18:08:06 -08001108#define mem_cgroup_from_res_counter(counter, member) \
1109 container_of(counter, struct mem_cgroup, member)
1110
Daisuke Nishimurab85a96c2009-01-07 18:08:12 -08001111static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
1112{
1113 if (do_swap_account) {
1114 if (res_counter_check_under_limit(&mem->res) &&
1115 res_counter_check_under_limit(&mem->memsw))
1116 return true;
1117 } else
1118 if (res_counter_check_under_limit(&mem->res))
1119 return true;
1120 return false;
1121}
1122
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001123static unsigned int get_swappiness(struct mem_cgroup *memcg)
1124{
1125 struct cgroup *cgrp = memcg->css.cgroup;
1126 unsigned int swappiness;
1127
1128 /* root ? */
1129 if (cgrp->parent == NULL)
1130 return vm_swappiness;
1131
1132 spin_lock(&memcg->reclaim_param_lock);
1133 swappiness = memcg->swappiness;
1134 spin_unlock(&memcg->reclaim_param_lock);
1135
1136 return swappiness;
1137}
1138
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001139static void mem_cgroup_start_move(struct mem_cgroup *mem)
1140{
1141 int cpu;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001142
1143 get_online_cpus();
1144 spin_lock(&mem->pcp_counter_lock);
1145 for_each_online_cpu(cpu)
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001146 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001147 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
1148 spin_unlock(&mem->pcp_counter_lock);
1149 put_online_cpus();
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001150
1151 synchronize_rcu();
1152}
1153
1154static void mem_cgroup_end_move(struct mem_cgroup *mem)
1155{
1156 int cpu;
1157
1158 if (!mem)
1159 return;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001160 get_online_cpus();
1161 spin_lock(&mem->pcp_counter_lock);
1162 for_each_online_cpu(cpu)
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001163 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001164 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
1165 spin_unlock(&mem->pcp_counter_lock);
1166 put_online_cpus();
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001167}
1168/*
1169 * 2 routines for checking "mem" is under move_account() or not.
1170 *
1171 * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used
1172 * for avoiding race in accounting. If true,
1173 * pc->mem_cgroup may be overwritten.
1174 *
1175 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
1176 * under hierarchy of moving cgroups. This is for
1177 * waiting at hith-memory prressure caused by "move".
1178 */
1179
1180static bool mem_cgroup_stealed(struct mem_cgroup *mem)
1181{
1182 VM_BUG_ON(!rcu_read_lock_held());
1183 return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
1184}
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001185
1186static bool mem_cgroup_under_move(struct mem_cgroup *mem)
1187{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001188 struct mem_cgroup *from;
1189 struct mem_cgroup *to;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001190 bool ret = false;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001191 /*
1192 * Unlike task_move routines, we access mc.to, mc.from not under
1193 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1194 */
1195 spin_lock(&mc.lock);
1196 from = mc.from;
1197 to = mc.to;
1198 if (!from)
1199 goto unlock;
1200 if (from == mem || to == mem
1201 || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css))
1202 || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css)))
1203 ret = true;
1204unlock:
1205 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001206 return ret;
1207}
1208
1209static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem)
1210{
1211 if (mc.moving_task && current != mc.moving_task) {
1212 if (mem_cgroup_under_move(mem)) {
1213 DEFINE_WAIT(wait);
1214 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1215 /* moving charge context might have finished. */
1216 if (mc.moving_task)
1217 schedule();
1218 finish_wait(&mc.waitq, &wait);
1219 return true;
1220 }
1221 }
1222 return false;
1223}
1224
Balbir Singhe2224322009-04-02 16:57:39 -07001225/**
Kirill A. Shutemov6a6135b2010-03-10 15:22:25 -08001226 * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
Balbir Singhe2224322009-04-02 16:57:39 -07001227 * @memcg: The memory cgroup that went over limit
1228 * @p: Task that is going to be killed
1229 *
1230 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1231 * enabled
1232 */
1233void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1234{
1235 struct cgroup *task_cgrp;
1236 struct cgroup *mem_cgrp;
1237 /*
1238 * Need a buffer in BSS, can't rely on allocations. The code relies
1239 * on the assumption that OOM is serialized for memory controller.
1240 * If this assumption is broken, revisit this code.
1241 */
1242 static char memcg_name[PATH_MAX];
1243 int ret;
1244
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -08001245 if (!memcg || !p)
Balbir Singhe2224322009-04-02 16:57:39 -07001246 return;
1247
1248
1249 rcu_read_lock();
1250
1251 mem_cgrp = memcg->css.cgroup;
1252 task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1253
1254 ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1255 if (ret < 0) {
1256 /*
1257 * Unfortunately, we are unable to convert to a useful name
1258 * But we'll still print out the usage information
1259 */
1260 rcu_read_unlock();
1261 goto done;
1262 }
1263 rcu_read_unlock();
1264
1265 printk(KERN_INFO "Task in %s killed", memcg_name);
1266
1267 rcu_read_lock();
1268 ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1269 if (ret < 0) {
1270 rcu_read_unlock();
1271 goto done;
1272 }
1273 rcu_read_unlock();
1274
1275 /*
1276 * Continues from above, so we don't need an KERN_ level
1277 */
1278 printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
1279done:
1280
1281 printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
1282 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1283 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1284 res_counter_read_u64(&memcg->res, RES_FAILCNT));
1285 printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
1286 "failcnt %llu\n",
1287 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1288 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1289 res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1290}
1291
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001292/*
1293 * This function returns the number of memcg under hierarchy tree. Returns
1294 * 1(self count) if no children.
1295 */
1296static int mem_cgroup_count_children(struct mem_cgroup *mem)
1297{
1298 int num = 0;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001299 struct mem_cgroup *iter;
1300
1301 for_each_mem_cgroup_tree(iter, mem)
1302 num++;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001303 return num;
1304}
1305
Balbir Singh6d61ef42009-01-07 18:08:06 -08001306/*
David Rientjesa63d83f2010-08-09 17:19:46 -07001307 * Return the memory (and swap, if configured) limit for a memcg.
1308 */
1309u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
1310{
1311 u64 limit;
1312 u64 memsw;
1313
Johannes Weinerf3e8eb72011-01-13 15:47:39 -08001314 limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
1315 limit += total_swap_pages << PAGE_SHIFT;
1316
David Rientjesa63d83f2010-08-09 17:19:46 -07001317 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1318 /*
1319 * If memsw is finite and limits the amount of swap space available
1320 * to this memcg, return that limit.
1321 */
1322 return min(limit, memsw);
1323}
1324
1325/*
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001326 * Visit the first child (need not be the first child as per the ordering
1327 * of the cgroup list, since we track last_scanned_child) of @mem and use
1328 * that to reclaim free pages from.
1329 */
1330static struct mem_cgroup *
1331mem_cgroup_select_victim(struct mem_cgroup *root_mem)
1332{
1333 struct mem_cgroup *ret = NULL;
1334 struct cgroup_subsys_state *css;
1335 int nextid, found;
1336
1337 if (!root_mem->use_hierarchy) {
1338 css_get(&root_mem->css);
1339 ret = root_mem;
1340 }
1341
1342 while (!ret) {
1343 rcu_read_lock();
1344 nextid = root_mem->last_scanned_child + 1;
1345 css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
1346 &found);
1347 if (css && css_tryget(css))
1348 ret = container_of(css, struct mem_cgroup, css);
1349
1350 rcu_read_unlock();
1351 /* Updates scanning parameter */
1352 spin_lock(&root_mem->reclaim_param_lock);
1353 if (!css) {
1354 /* this means start scan from ID:1 */
1355 root_mem->last_scanned_child = 0;
1356 } else
1357 root_mem->last_scanned_child = found;
1358 spin_unlock(&root_mem->reclaim_param_lock);
1359 }
1360
1361 return ret;
1362}
1363
1364/*
1365 * Scan the hierarchy if needed to reclaim memory. We remember the last child
1366 * we reclaimed from, so that we don't end up penalizing one child extensively
1367 * based on its position in the children list.
Balbir Singh6d61ef42009-01-07 18:08:06 -08001368 *
1369 * root_mem is the original ancestor that we've been reclaim from.
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001370 *
1371 * We give up and return to the caller when we visit root_mem twice.
1372 * (other groups can be removed while we're walking....)
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001373 *
1374 * If shrink==true, for avoiding to free too much, this returns immedieately.
Balbir Singh6d61ef42009-01-07 18:08:06 -08001375 */
1376static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
Balbir Singh4e416952009-09-23 15:56:39 -07001377 struct zone *zone,
Balbir Singh75822b42009-09-23 15:56:38 -07001378 gfp_t gfp_mask,
1379 unsigned long reclaim_options)
Balbir Singh6d61ef42009-01-07 18:08:06 -08001380{
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001381 struct mem_cgroup *victim;
1382 int ret, total = 0;
1383 int loop = 0;
Balbir Singh75822b42009-09-23 15:56:38 -07001384 bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
1385 bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
Balbir Singh4e416952009-09-23 15:56:39 -07001386 bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
1387 unsigned long excess = mem_cgroup_get_excess(root_mem);
Balbir Singh6d61ef42009-01-07 18:08:06 -08001388
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07001389 /* If memsw_is_minimum==1, swap-out is of-no-use. */
1390 if (root_mem->memsw_is_minimum)
1391 noswap = true;
1392
Balbir Singh4e416952009-09-23 15:56:39 -07001393 while (1) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001394 victim = mem_cgroup_select_victim(root_mem);
Balbir Singh4e416952009-09-23 15:56:39 -07001395 if (victim == root_mem) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001396 loop++;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001397 if (loop >= 1)
1398 drain_all_stock_async();
Balbir Singh4e416952009-09-23 15:56:39 -07001399 if (loop >= 2) {
1400 /*
1401 * If we have not been able to reclaim
1402 * anything, it might because there are
1403 * no reclaimable pages under this hierarchy
1404 */
1405 if (!check_soft || !total) {
1406 css_put(&victim->css);
1407 break;
1408 }
1409 /*
1410 * We want to do more targetted reclaim.
1411 * excess >> 2 is not to excessive so as to
1412 * reclaim too much, nor too less that we keep
1413 * coming back to reclaim from this cgroup
1414 */
1415 if (total >= (excess >> 2) ||
1416 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
1417 css_put(&victim->css);
1418 break;
1419 }
1420 }
1421 }
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08001422 if (!mem_cgroup_local_usage(victim)) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001423 /* this cgroup's local usage == 0 */
1424 css_put(&victim->css);
Balbir Singh6d61ef42009-01-07 18:08:06 -08001425 continue;
1426 }
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001427 /* we use swappiness of local cgroup */
Balbir Singh4e416952009-09-23 15:56:39 -07001428 if (check_soft)
1429 ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
KOSAKI Motohiro14fec792010-08-10 18:03:05 -07001430 noswap, get_swappiness(victim), zone);
Balbir Singh4e416952009-09-23 15:56:39 -07001431 else
1432 ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
1433 noswap, get_swappiness(victim));
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001434 css_put(&victim->css);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001435 /*
1436 * At shrinking usage, we can't check we should stop here or
1437 * reclaim more. It's depends on callers. last_scanned_child
1438 * will work enough for keeping fairness under tree.
1439 */
1440 if (shrink)
1441 return ret;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001442 total += ret;
Balbir Singh4e416952009-09-23 15:56:39 -07001443 if (check_soft) {
1444 if (res_counter_check_under_soft_limit(&root_mem->res))
1445 return total;
1446 } else if (mem_cgroup_check_under_limit(root_mem))
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001447 return 1 + total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001448 }
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001449 return total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001450}
1451
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001452/*
1453 * Check OOM-Killer is already running under our hierarchy.
1454 * If someone is running, return false.
1455 */
1456static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
1457{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001458 int x, lock_count = 0;
1459 struct mem_cgroup *iter;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001460
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001461 for_each_mem_cgroup_tree(iter, mem) {
1462 x = atomic_inc_return(&iter->oom_lock);
1463 lock_count = max(x, lock_count);
1464 }
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001465
1466 if (lock_count == 1)
1467 return true;
1468 return false;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001469}
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001470
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001471static int mem_cgroup_oom_unlock(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001472{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001473 struct mem_cgroup *iter;
1474
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001475 /*
1476 * When a new child is created while the hierarchy is under oom,
1477 * mem_cgroup_oom_lock() may not be called. We have to use
1478 * atomic_add_unless() here.
1479 */
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001480 for_each_mem_cgroup_tree(iter, mem)
1481 atomic_add_unless(&iter->oom_lock, -1, 0);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001482 return 0;
1483}
1484
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001485
1486static DEFINE_MUTEX(memcg_oom_mutex);
1487static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1488
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001489struct oom_wait_info {
1490 struct mem_cgroup *mem;
1491 wait_queue_t wait;
1492};
1493
1494static int memcg_oom_wake_function(wait_queue_t *wait,
1495 unsigned mode, int sync, void *arg)
1496{
1497 struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg;
1498 struct oom_wait_info *oom_wait_info;
1499
1500 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1501
1502 if (oom_wait_info->mem == wake_mem)
1503 goto wakeup;
1504 /* if no hierarchy, no match */
1505 if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy)
1506 return 0;
1507 /*
1508 * Both of oom_wait_info->mem and wake_mem are stable under us.
1509 * Then we can use css_is_ancestor without taking care of RCU.
1510 */
1511 if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) &&
1512 !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css))
1513 return 0;
1514
1515wakeup:
1516 return autoremove_wake_function(wait, mode, sync, arg);
1517}
1518
1519static void memcg_wakeup_oom(struct mem_cgroup *mem)
1520{
1521 /* for filtering, pass "mem" as argument. */
1522 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem);
1523}
1524
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001525static void memcg_oom_recover(struct mem_cgroup *mem)
1526{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001527 if (mem && atomic_read(&mem->oom_lock))
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001528 memcg_wakeup_oom(mem);
1529}
1530
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001531/*
1532 * try to call OOM killer. returns false if we should exit memory-reclaim loop.
1533 */
1534bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
1535{
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001536 struct oom_wait_info owait;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001537 bool locked, need_to_kill;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001538
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001539 owait.mem = mem;
1540 owait.wait.flags = 0;
1541 owait.wait.func = memcg_oom_wake_function;
1542 owait.wait.private = current;
1543 INIT_LIST_HEAD(&owait.wait.task_list);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001544 need_to_kill = true;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001545 /* At first, try to OOM lock hierarchy under mem.*/
1546 mutex_lock(&memcg_oom_mutex);
1547 locked = mem_cgroup_oom_lock(mem);
1548 /*
1549 * Even if signal_pending(), we can't quit charge() loop without
1550 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
1551 * under OOM is always welcomed, use TASK_KILLABLE here.
1552 */
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001553 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1554 if (!locked || mem->oom_kill_disable)
1555 need_to_kill = false;
1556 if (locked)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07001557 mem_cgroup_oom_notify(mem);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001558 mutex_unlock(&memcg_oom_mutex);
1559
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001560 if (need_to_kill) {
1561 finish_wait(&memcg_oom_waitq, &owait.wait);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001562 mem_cgroup_out_of_memory(mem, mask);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001563 } else {
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001564 schedule();
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001565 finish_wait(&memcg_oom_waitq, &owait.wait);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001566 }
1567 mutex_lock(&memcg_oom_mutex);
1568 mem_cgroup_oom_unlock(mem);
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001569 memcg_wakeup_oom(mem);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001570 mutex_unlock(&memcg_oom_mutex);
1571
1572 if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
1573 return false;
1574 /* Give chance to dying process */
1575 schedule_timeout(1);
1576 return true;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001577}
1578
Balbir Singhd69b0422009-06-17 16:26:34 -07001579/*
1580 * Currently used to update mapped file statistics, but the routine can be
1581 * generalized to update other statistics as well.
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001582 *
1583 * Notes: Race condition
1584 *
1585 * We usually use page_cgroup_lock() for accessing page_cgroup member but
1586 * it tends to be costly. But considering some conditions, we doesn't need
1587 * to do so _always_.
1588 *
1589 * Considering "charge", lock_page_cgroup() is not required because all
1590 * file-stat operations happen after a page is attached to radix-tree. There
1591 * are no race with "charge".
1592 *
1593 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
1594 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
1595 * if there are race with "uncharge". Statistics itself is properly handled
1596 * by flags.
1597 *
1598 * Considering "move", this is an only case we see a race. To make the race
1599 * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are
1600 * possibility of race condition. If there is, we take a lock.
Balbir Singhd69b0422009-06-17 16:26:34 -07001601 */
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07001602
Greg Thelen2a7106f2011-01-13 15:47:37 -08001603void mem_cgroup_update_page_stat(struct page *page,
1604 enum mem_cgroup_page_stat_item idx, int val)
Balbir Singhd69b0422009-06-17 16:26:34 -07001605{
1606 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001607 struct page_cgroup *pc = lookup_page_cgroup(page);
1608 bool need_unlock = false;
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08001609 unsigned long uninitialized_var(flags);
Balbir Singhd69b0422009-06-17 16:26:34 -07001610
Balbir Singhd69b0422009-06-17 16:26:34 -07001611 if (unlikely(!pc))
1612 return;
1613
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001614 rcu_read_lock();
Balbir Singhd69b0422009-06-17 16:26:34 -07001615 mem = pc->mem_cgroup;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001616 if (unlikely(!mem || !PageCgroupUsed(pc)))
1617 goto out;
1618 /* pc->mem_cgroup is unstable ? */
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08001619 if (unlikely(mem_cgroup_stealed(mem)) || PageTransHuge(page)) {
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001620 /* take a lock against to access pc->mem_cgroup */
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08001621 move_lock_page_cgroup(pc, &flags);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001622 need_unlock = true;
1623 mem = pc->mem_cgroup;
1624 if (!mem || !PageCgroupUsed(pc))
1625 goto out;
1626 }
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07001627
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07001628 switch (idx) {
Greg Thelen2a7106f2011-01-13 15:47:37 -08001629 case MEMCG_NR_FILE_MAPPED:
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07001630 if (val > 0)
1631 SetPageCgroupFileMapped(pc);
1632 else if (!page_mapped(page))
KAMEZAWA Hiroyuki0c270f82010-10-27 15:33:39 -07001633 ClearPageCgroupFileMapped(pc);
Greg Thelen2a7106f2011-01-13 15:47:37 -08001634 idx = MEM_CGROUP_STAT_FILE_MAPPED;
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07001635 break;
1636 default:
1637 BUG();
KAMEZAWA Hiroyuki8725d542010-04-06 14:35:05 -07001638 }
Balbir Singhd69b0422009-06-17 16:26:34 -07001639
Greg Thelen2a7106f2011-01-13 15:47:37 -08001640 this_cpu_add(mem->stat->count[idx], val);
1641
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001642out:
1643 if (unlikely(need_unlock))
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08001644 move_unlock_page_cgroup(pc, &flags);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001645 rcu_read_unlock();
1646 return;
Balbir Singhd69b0422009-06-17 16:26:34 -07001647}
Greg Thelen2a7106f2011-01-13 15:47:37 -08001648EXPORT_SYMBOL(mem_cgroup_update_page_stat);
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07001649
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001650/*
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001651 * size of first charge trial. "32" comes from vmscan.c's magic value.
1652 * TODO: maybe necessary to use big numbers in big irons.
1653 */
1654#define CHARGE_SIZE (32 * PAGE_SIZE)
1655struct memcg_stock_pcp {
1656 struct mem_cgroup *cached; /* this never be root cgroup */
1657 int charge;
1658 struct work_struct work;
1659};
1660static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1661static atomic_t memcg_drain_count;
1662
1663/*
1664 * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed
1665 * from local stock and true is returned. If the stock is 0 or charges from a
1666 * cgroup which is not current target, returns false. This stock will be
1667 * refilled.
1668 */
1669static bool consume_stock(struct mem_cgroup *mem)
1670{
1671 struct memcg_stock_pcp *stock;
1672 bool ret = true;
1673
1674 stock = &get_cpu_var(memcg_stock);
1675 if (mem == stock->cached && stock->charge)
1676 stock->charge -= PAGE_SIZE;
1677 else /* need to call res_counter_charge */
1678 ret = false;
1679 put_cpu_var(memcg_stock);
1680 return ret;
1681}
1682
1683/*
1684 * Returns stocks cached in percpu to res_counter and reset cached information.
1685 */
1686static void drain_stock(struct memcg_stock_pcp *stock)
1687{
1688 struct mem_cgroup *old = stock->cached;
1689
1690 if (stock->charge) {
1691 res_counter_uncharge(&old->res, stock->charge);
1692 if (do_swap_account)
1693 res_counter_uncharge(&old->memsw, stock->charge);
1694 }
1695 stock->cached = NULL;
1696 stock->charge = 0;
1697}
1698
1699/*
1700 * This must be called under preempt disabled or must be called by
1701 * a thread which is pinned to local cpu.
1702 */
1703static void drain_local_stock(struct work_struct *dummy)
1704{
1705 struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
1706 drain_stock(stock);
1707}
1708
1709/*
1710 * Cache charges(val) which is from res_counter, to local per_cpu area.
Greg Thelen320cc512010-03-15 15:27:28 +01001711 * This will be consumed by consume_stock() function, later.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001712 */
1713static void refill_stock(struct mem_cgroup *mem, int val)
1714{
1715 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
1716
1717 if (stock->cached != mem) { /* reset if necessary */
1718 drain_stock(stock);
1719 stock->cached = mem;
1720 }
1721 stock->charge += val;
1722 put_cpu_var(memcg_stock);
1723}
1724
1725/*
1726 * Tries to drain stocked charges in other cpus. This function is asynchronous
1727 * and just put a work per cpu for draining localy on each cpu. Caller can
1728 * expects some charges will be back to res_counter later but cannot wait for
1729 * it.
1730 */
1731static void drain_all_stock_async(void)
1732{
1733 int cpu;
1734 /* This function is for scheduling "drain" in asynchronous way.
1735 * The result of "drain" is not directly handled by callers. Then,
1736 * if someone is calling drain, we don't have to call drain more.
1737 * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if
1738 * there is a race. We just do loose check here.
1739 */
1740 if (atomic_read(&memcg_drain_count))
1741 return;
1742 /* Notify other cpus that system-wide "drain" is running */
1743 atomic_inc(&memcg_drain_count);
1744 get_online_cpus();
1745 for_each_online_cpu(cpu) {
1746 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1747 schedule_work_on(cpu, &stock->work);
1748 }
1749 put_online_cpus();
1750 atomic_dec(&memcg_drain_count);
1751 /* We don't wait for flush_work */
1752}
1753
1754/* This is a synchronous drain interface. */
1755static void drain_all_stock_sync(void)
1756{
1757 /* called when force_empty is called */
1758 atomic_inc(&memcg_drain_count);
1759 schedule_on_each_cpu(drain_local_stock);
1760 atomic_dec(&memcg_drain_count);
1761}
1762
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001763/*
1764 * This function drains percpu counter value from DEAD cpu and
1765 * move it to local cpu. Note that this function can be preempted.
1766 */
1767static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu)
1768{
1769 int i;
1770
1771 spin_lock(&mem->pcp_counter_lock);
1772 for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) {
1773 s64 x = per_cpu(mem->stat->count[i], cpu);
1774
1775 per_cpu(mem->stat->count[i], cpu) = 0;
1776 mem->nocpu_base.count[i] += x;
1777 }
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001778 /* need to clear ON_MOVE value, works as a kind of lock. */
1779 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
1780 spin_unlock(&mem->pcp_counter_lock);
1781}
1782
1783static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu)
1784{
1785 int idx = MEM_CGROUP_ON_MOVE;
1786
1787 spin_lock(&mem->pcp_counter_lock);
1788 per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx];
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001789 spin_unlock(&mem->pcp_counter_lock);
1790}
1791
1792static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001793 unsigned long action,
1794 void *hcpu)
1795{
1796 int cpu = (unsigned long)hcpu;
1797 struct memcg_stock_pcp *stock;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001798 struct mem_cgroup *iter;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001799
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001800 if ((action == CPU_ONLINE)) {
1801 for_each_mem_cgroup_all(iter)
1802 synchronize_mem_cgroup_on_move(iter, cpu);
1803 return NOTIFY_OK;
1804 }
1805
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001806 if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001807 return NOTIFY_OK;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001808
1809 for_each_mem_cgroup_all(iter)
1810 mem_cgroup_drain_pcp_counter(iter, cpu);
1811
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001812 stock = &per_cpu(memcg_stock, cpu);
1813 drain_stock(stock);
1814 return NOTIFY_OK;
1815}
1816
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001817
1818/* See __mem_cgroup_try_charge() for details */
1819enum {
1820 CHARGE_OK, /* success */
1821 CHARGE_RETRY, /* need to retry but retry is not bad */
1822 CHARGE_NOMEM, /* we can't do more. return -ENOMEM */
1823 CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */
1824 CHARGE_OOM_DIE, /* the current is killed because of OOM */
1825};
1826
1827static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
1828 int csize, bool oom_check)
1829{
1830 struct mem_cgroup *mem_over_limit;
1831 struct res_counter *fail_res;
1832 unsigned long flags = 0;
1833 int ret;
1834
1835 ret = res_counter_charge(&mem->res, csize, &fail_res);
1836
1837 if (likely(!ret)) {
1838 if (!do_swap_account)
1839 return CHARGE_OK;
1840 ret = res_counter_charge(&mem->memsw, csize, &fail_res);
1841 if (likely(!ret))
1842 return CHARGE_OK;
1843
1844 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
1845 flags |= MEM_CGROUP_RECLAIM_NOSWAP;
1846 } else
1847 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
1848
1849 if (csize > PAGE_SIZE) /* change csize and retry */
1850 return CHARGE_RETRY;
1851
1852 if (!(gfp_mask & __GFP_WAIT))
1853 return CHARGE_WOULDBLOCK;
1854
1855 ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
1856 gfp_mask, flags);
1857 /*
1858 * try_to_free_mem_cgroup_pages() might not give us a full
1859 * picture of reclaim. Some pages are reclaimed and might be
1860 * moved to swap cache or just unmapped from the cgroup.
1861 * Check the limit again to see if the reclaim reduced the
1862 * current usage of the cgroup before giving up
1863 */
1864 if (ret || mem_cgroup_check_under_limit(mem_over_limit))
1865 return CHARGE_RETRY;
1866
1867 /*
1868 * At task move, charge accounts can be doubly counted. So, it's
1869 * better to wait until the end of task_move if something is going on.
1870 */
1871 if (mem_cgroup_wait_acct_move(mem_over_limit))
1872 return CHARGE_RETRY;
1873
1874 /* If we don't need to call oom-killer at el, return immediately */
1875 if (!oom_check)
1876 return CHARGE_NOMEM;
1877 /* check OOM */
1878 if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask))
1879 return CHARGE_OOM_DIE;
1880
1881 return CHARGE_RETRY;
1882}
1883
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001884/*
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001885 * Unlike exported interface, "oom" parameter is added. if oom==true,
1886 * oom-killer can be invoked.
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001887 */
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001888static int __mem_cgroup_try_charge(struct mm_struct *mm,
Andrea Arcangeliec168512011-01-13 15:46:56 -08001889 gfp_t gfp_mask,
1890 struct mem_cgroup **memcg, bool oom,
1891 int page_size)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001892{
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001893 int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
1894 struct mem_cgroup *mem = NULL;
1895 int ret;
Andrea Arcangeliec168512011-01-13 15:46:56 -08001896 int csize = max(CHARGE_SIZE, (unsigned long) page_size);
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001897
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001898 /*
1899 * Unlike gloval-vm's OOM-kill, we're not in memory shortage
1900 * in system level. So, allow to go ahead dying process in addition to
1901 * MEMDIE process.
1902 */
1903 if (unlikely(test_thread_flag(TIF_MEMDIE)
1904 || fatal_signal_pending(current)))
1905 goto bypass;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001906
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001907 /*
Hugh Dickins3be91272008-02-07 00:14:19 -08001908 * We always charge the cgroup the mm_struct belongs to.
1909 * The mm_struct's mem_cgroup changes on task migration if the
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001910 * thread group leader migrates. It's possible that mm is not
1911 * set, if so charge the init_mm (happens for pagecache usage).
1912 */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001913 if (!*memcg && !mm)
1914 goto bypass;
1915again:
1916 if (*memcg) { /* css should be a valid one */
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001917 mem = *memcg;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001918 VM_BUG_ON(css_is_removed(&mem->css));
1919 if (mem_cgroup_is_root(mem))
1920 goto done;
Andrea Arcangeliec168512011-01-13 15:46:56 -08001921 if (page_size == PAGE_SIZE && consume_stock(mem))
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001922 goto done;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001923 css_get(&mem->css);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001924 } else {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001925 struct task_struct *p;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001926
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001927 rcu_read_lock();
1928 p = rcu_dereference(mm->owner);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001929 /*
KAMEZAWA Hiroyukiebb76ce2010-12-29 14:07:11 -08001930 * Because we don't have task_lock(), "p" can exit.
1931 * In that case, "mem" can point to root or p can be NULL with
1932 * race with swapoff. Then, we have small risk of mis-accouning.
1933 * But such kind of mis-account by race always happens because
1934 * we don't have cgroup_mutex(). It's overkill and we allo that
1935 * small race, here.
1936 * (*) swapoff at el will charge against mm-struct not against
1937 * task-struct. So, mm->owner can be NULL.
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001938 */
1939 mem = mem_cgroup_from_task(p);
KAMEZAWA Hiroyukiebb76ce2010-12-29 14:07:11 -08001940 if (!mem || mem_cgroup_is_root(mem)) {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001941 rcu_read_unlock();
1942 goto done;
1943 }
Andrea Arcangeliec168512011-01-13 15:46:56 -08001944 if (page_size == PAGE_SIZE && consume_stock(mem)) {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001945 /*
1946 * It seems dagerous to access memcg without css_get().
1947 * But considering how consume_stok works, it's not
1948 * necessary. If consume_stock success, some charges
1949 * from this memcg are cached on this cpu. So, we
1950 * don't need to call css_get()/css_tryget() before
1951 * calling consume_stock().
1952 */
1953 rcu_read_unlock();
1954 goto done;
1955 }
1956 /* after here, we may be blocked. we need to get refcnt */
1957 if (!css_tryget(&mem->css)) {
1958 rcu_read_unlock();
1959 goto again;
1960 }
1961 rcu_read_unlock();
1962 }
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001963
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001964 do {
1965 bool oom_check;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001966
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001967 /* If killed, bypass charge */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001968 if (fatal_signal_pending(current)) {
1969 css_put(&mem->css);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001970 goto bypass;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001971 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001972
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001973 oom_check = false;
1974 if (oom && !nr_oom_retries) {
1975 oom_check = true;
1976 nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
1977 }
Balbir Singh6d61ef42009-01-07 18:08:06 -08001978
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001979 ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check);
1980
1981 switch (ret) {
1982 case CHARGE_OK:
1983 break;
1984 case CHARGE_RETRY: /* not in OOM situation but retry */
Andrea Arcangeliec168512011-01-13 15:46:56 -08001985 csize = page_size;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001986 css_put(&mem->css);
1987 mem = NULL;
1988 goto again;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001989 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001990 css_put(&mem->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001991 goto nomem;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001992 case CHARGE_NOMEM: /* OOM routine works */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001993 if (!oom) {
1994 css_put(&mem->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001995 goto nomem;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001996 }
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001997 /* If oom, we never return -ENOMEM */
1998 nr_oom_retries--;
1999 break;
2000 case CHARGE_OOM_DIE: /* Killed by OOM Killer */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002001 css_put(&mem->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002002 goto bypass;
Balbir Singh66e17072008-02-07 00:13:56 -08002003 }
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002004 } while (ret != CHARGE_OK);
2005
Andrea Arcangeliec168512011-01-13 15:46:56 -08002006 if (csize > page_size)
2007 refill_stock(mem, csize - page_size);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002008 css_put(&mem->css);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002009done:
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002010 *memcg = mem;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002011 return 0;
2012nomem:
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002013 *memcg = NULL;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002014 return -ENOMEM;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002015bypass:
2016 *memcg = NULL;
2017 return 0;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002018}
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002019
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002020/*
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002021 * Somemtimes we have to undo a charge we got by try_charge().
2022 * This function is for that and do uncharge, put css's refcnt.
2023 * gotten by try_charge().
2024 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002025static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
2026 unsigned long count)
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002027{
2028 if (!mem_cgroup_is_root(mem)) {
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002029 res_counter_uncharge(&mem->res, PAGE_SIZE * count);
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002030 if (do_swap_account)
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002031 res_counter_uncharge(&mem->memsw, PAGE_SIZE * count);
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002032 }
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002033}
2034
Andrea Arcangeliec168512011-01-13 15:46:56 -08002035static void mem_cgroup_cancel_charge(struct mem_cgroup *mem,
2036 int page_size)
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002037{
Andrea Arcangeliec168512011-01-13 15:46:56 -08002038 __mem_cgroup_cancel_charge(mem, page_size >> PAGE_SHIFT);
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002039}
2040
2041/*
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002042 * A helper function to get mem_cgroup from ID. must be called under
2043 * rcu_read_lock(). The caller must check css_is_removed() or some if
2044 * it's concern. (dropping refcnt from swap can be called against removed
2045 * memcg.)
2046 */
2047static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
2048{
2049 struct cgroup_subsys_state *css;
2050
2051 /* ID 0 is unused ID */
2052 if (!id)
2053 return NULL;
2054 css = css_lookup(&mem_cgroup_subsys, id);
2055 if (!css)
2056 return NULL;
2057 return container_of(css, struct mem_cgroup, css);
2058}
2059
Wu Fengguange42d9d52009-12-16 12:19:59 +01002060struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002061{
Wu Fengguange42d9d52009-12-16 12:19:59 +01002062 struct mem_cgroup *mem = NULL;
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002063 struct page_cgroup *pc;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002064 unsigned short id;
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002065 swp_entry_t ent;
2066
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002067 VM_BUG_ON(!PageLocked(page));
2068
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002069 pc = lookup_page_cgroup(page);
Daisuke Nishimurac0bd3f62009-04-30 15:08:11 -07002070 lock_page_cgroup(pc);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002071 if (PageCgroupUsed(pc)) {
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002072 mem = pc->mem_cgroup;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002073 if (mem && !css_tryget(&mem->css))
2074 mem = NULL;
Wu Fengguange42d9d52009-12-16 12:19:59 +01002075 } else if (PageSwapCache(page)) {
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002076 ent.val = page_private(page);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002077 id = lookup_swap_cgroup(ent);
2078 rcu_read_lock();
2079 mem = mem_cgroup_lookup(id);
2080 if (mem && !css_tryget(&mem->css))
2081 mem = NULL;
2082 rcu_read_unlock();
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002083 }
Daisuke Nishimurac0bd3f62009-04-30 15:08:11 -07002084 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002085 return mem;
2086}
2087
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002088static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
2089 struct page_cgroup *pc,
2090 enum charge_type ctype,
2091 int page_size)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002092{
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002093 int nr_pages = page_size >> PAGE_SHIFT;
2094
2095 /* try_charge() can return NULL to *memcg, taking care of it. */
2096 if (!mem)
2097 return;
2098
2099 lock_page_cgroup(pc);
2100 if (unlikely(PageCgroupUsed(pc))) {
2101 unlock_page_cgroup(pc);
2102 mem_cgroup_cancel_charge(mem, page_size);
2103 return;
2104 }
2105 /*
2106 * we don't need page_cgroup_lock about tail pages, becase they are not
2107 * accessed by any other context at this point.
2108 */
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002109 pc->mem_cgroup = mem;
KAMEZAWA Hiroyuki261fb612009-09-23 15:56:33 -07002110 /*
2111 * We access a page_cgroup asynchronously without lock_page_cgroup().
2112 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
2113 * is accessed after testing USED bit. To make pc->mem_cgroup visible
2114 * before USED bit, we need memory barrier here.
2115 * See mem_cgroup_add_lru_list(), etc.
2116 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002117 smp_wmb();
Balbir Singh4b3bde42009-09-23 15:56:32 -07002118 switch (ctype) {
2119 case MEM_CGROUP_CHARGE_TYPE_CACHE:
2120 case MEM_CGROUP_CHARGE_TYPE_SHMEM:
2121 SetPageCgroupCache(pc);
2122 SetPageCgroupUsed(pc);
2123 break;
2124 case MEM_CGROUP_CHARGE_TYPE_MAPPED:
2125 ClearPageCgroupCache(pc);
2126 SetPageCgroupUsed(pc);
2127 break;
2128 default:
2129 break;
2130 }
Hugh Dickins3be91272008-02-07 00:14:19 -08002131
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002132 mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), nr_pages);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002133 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki430e48632010-03-10 15:22:30 -08002134 /*
2135 * "charge_statistics" updated event counter. Then, check it.
2136 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
2137 * if they exceeds softlimit.
2138 */
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08002139 memcg_check_events(mem, pc->page);
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002140}
2141
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002142#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2143
2144#define PCGF_NOCOPY_AT_SPLIT ((1 << PCG_LOCK) | (1 << PCG_MOVE_LOCK) |\
2145 (1 << PCG_ACCT_LRU) | (1 << PCG_MIGRATION))
2146/*
2147 * Because tail pages are not marked as "used", set it. We're under
2148 * zone->lru_lock, 'splitting on pmd' and compund_lock.
2149 */
2150void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail)
2151{
2152 struct page_cgroup *head_pc = lookup_page_cgroup(head);
2153 struct page_cgroup *tail_pc = lookup_page_cgroup(tail);
2154 unsigned long flags;
2155
2156 /*
KAMEZAWA Hiroyukiece35ca2011-01-20 14:44:24 -08002157 * We have no races with charge/uncharge but will have races with
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002158 * page state accounting.
2159 */
2160 move_lock_page_cgroup(head_pc, &flags);
2161
2162 tail_pc->mem_cgroup = head_pc->mem_cgroup;
2163 smp_wmb(); /* see __commit_charge() */
KAMEZAWA Hiroyukiece35ca2011-01-20 14:44:24 -08002164 if (PageCgroupAcctLRU(head_pc)) {
2165 enum lru_list lru;
2166 struct mem_cgroup_per_zone *mz;
2167
2168 /*
2169 * LRU flags cannot be copied because we need to add tail
2170 *.page to LRU by generic call and our hook will be called.
2171 * We hold lru_lock, then, reduce counter directly.
2172 */
2173 lru = page_lru(head);
2174 mz = page_cgroup_zoneinfo(head_pc);
2175 MEM_CGROUP_ZSTAT(mz, lru) -= 1;
2176 }
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002177 tail_pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
2178 move_unlock_page_cgroup(head_pc, &flags);
2179}
2180#endif
2181
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002182/**
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002183 * __mem_cgroup_move_account - move account of the page
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002184 * @pc: page_cgroup of the page.
2185 * @from: mem_cgroup which the page is moved from.
2186 * @to: mem_cgroup which the page is moved to. @from != @to.
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002187 * @uncharge: whether we should call uncharge and css_put against @from.
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002188 *
2189 * The caller must confirm following.
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002190 * - page is not on LRU (isolate_page() is useful.)
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002191 * - the pc is locked, used, and ->mem_cgroup points to @from.
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002192 *
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002193 * This function doesn't do "charge" nor css_get to new cgroup. It should be
2194 * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is
2195 * true, this function does "uncharge" from old cgroup, but it doesn't if
2196 * @uncharge is false, so a caller should do "uncharge".
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002197 */
2198
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002199static void __mem_cgroup_move_account(struct page_cgroup *pc,
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002200 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002201{
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002202 VM_BUG_ON(from == to);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002203 VM_BUG_ON(PageLRU(pc->page));
Kirill A. Shutemov112bc2e2010-11-24 12:56:58 -08002204 VM_BUG_ON(!page_is_cgroup_locked(pc));
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002205 VM_BUG_ON(!PageCgroupUsed(pc));
2206 VM_BUG_ON(pc->mem_cgroup != from);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002207
KAMEZAWA Hiroyuki8725d542010-04-06 14:35:05 -07002208 if (PageCgroupFileMapped(pc)) {
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08002209 /* Update mapped_file data for mem_cgroup */
2210 preempt_disable();
2211 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
2212 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
2213 preempt_enable();
Balbir Singhd69b0422009-06-17 16:26:34 -07002214 }
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -08002215 mem_cgroup_charge_statistics(from, PageCgroupCache(pc), -1);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002216 if (uncharge)
2217 /* This is not "cancel", but cancel_charge does all we need. */
Andrea Arcangeliec168512011-01-13 15:46:56 -08002218 mem_cgroup_cancel_charge(from, PAGE_SIZE);
Balbir Singhd69b0422009-06-17 16:26:34 -07002219
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002220 /* caller should have done css_get */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002221 pc->mem_cgroup = to;
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -08002222 mem_cgroup_charge_statistics(to, PageCgroupCache(pc), 1);
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002223 /*
2224 * We charges against "to" which may not have any tasks. Then, "to"
2225 * can be under rmdir(). But in current implementation, caller of
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08002226 * this function is just force_empty() and move charge, so it's
2227 * garanteed that "to" is never removed. So, we don't check rmdir
2228 * status here.
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002229 */
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002230}
2231
2232/*
2233 * check whether the @pc is valid for moving account and call
2234 * __mem_cgroup_move_account()
2235 */
2236static int mem_cgroup_move_account(struct page_cgroup *pc,
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002237 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002238{
2239 int ret = -EINVAL;
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08002240 unsigned long flags;
2241
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002242 lock_page_cgroup(pc);
2243 if (PageCgroupUsed(pc) && pc->mem_cgroup == from) {
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08002244 move_lock_page_cgroup(pc, &flags);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002245 __mem_cgroup_move_account(pc, from, to, uncharge);
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08002246 move_unlock_page_cgroup(pc, &flags);
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002247 ret = 0;
2248 }
2249 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08002250 /*
2251 * check events
2252 */
2253 memcg_check_events(to, pc->page);
2254 memcg_check_events(from, pc->page);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002255 return ret;
2256}
2257
2258/*
2259 * move charges to its parent.
2260 */
2261
2262static int mem_cgroup_move_parent(struct page_cgroup *pc,
2263 struct mem_cgroup *child,
2264 gfp_t gfp_mask)
2265{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002266 struct page *page = pc->page;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002267 struct cgroup *cg = child->css.cgroup;
2268 struct cgroup *pcg = cg->parent;
2269 struct mem_cgroup *parent;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002270 int ret;
2271
2272 /* Is ROOT ? */
2273 if (!pcg)
2274 return -EINVAL;
2275
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002276 ret = -EBUSY;
2277 if (!get_page_unless_zero(page))
2278 goto out;
2279 if (isolate_lru_page(page))
2280 goto put;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002281
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002282 parent = mem_cgroup_from_cont(pcg);
Andrea Arcangeliec168512011-01-13 15:46:56 -08002283 ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false,
2284 PAGE_SIZE);
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002285 if (ret || !parent)
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002286 goto put_back;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002287
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002288 ret = mem_cgroup_move_account(pc, child, parent, true);
2289 if (ret)
Andrea Arcangeliec168512011-01-13 15:46:56 -08002290 mem_cgroup_cancel_charge(parent, PAGE_SIZE);
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002291put_back:
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002292 putback_lru_page(page);
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002293put:
Daisuke Nishimura40d58132009-01-15 13:51:12 -08002294 put_page(page);
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002295out:
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002296 return ret;
2297}
2298
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002299/*
2300 * Charge the memory controller for page usage.
2301 * Return
2302 * 0 if the charge was successful
2303 * < 0 if the cgroup is over its limit
2304 */
2305static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002306 gfp_t gfp_mask, enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002307{
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002308 struct mem_cgroup *mem = NULL;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002309 struct page_cgroup *pc;
2310 int ret;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002311 int page_size = PAGE_SIZE;
2312
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08002313 if (PageTransHuge(page)) {
Andrea Arcangeliec168512011-01-13 15:46:56 -08002314 page_size <<= compound_order(page);
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08002315 VM_BUG_ON(!PageTransHuge(page));
2316 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002317
2318 pc = lookup_page_cgroup(page);
2319 /* can happen at boot */
2320 if (unlikely(!pc))
2321 return 0;
2322 prefetchw(pc);
2323
Andrea Arcangeliec168512011-01-13 15:46:56 -08002324 ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true, page_size);
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002325 if (ret || !mem)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002326 return ret;
2327
Andrea Arcangeliec168512011-01-13 15:46:56 -08002328 __mem_cgroup_commit_charge(mem, pc, ctype, page_size);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002329 return 0;
2330}
2331
2332int mem_cgroup_newpage_charge(struct page *page,
2333 struct mm_struct *mm, gfp_t gfp_mask)
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08002334{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002335 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07002336 return 0;
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002337 /*
2338 * If already mapped, we don't have to account.
2339 * If page cache, page->mapping has address_space.
2340 * But page->mapping may have out-of-use anon_vma pointer,
2341 * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
2342 * is NULL.
2343 */
2344 if (page_mapped(page) || (page->mapping && !PageAnon(page)))
2345 return 0;
2346 if (unlikely(!mm))
2347 mm = &init_mm;
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08002348 return mem_cgroup_charge_common(page, mm, gfp_mask,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002349 MEM_CGROUP_CHARGE_TYPE_MAPPED);
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08002350}
2351
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002352static void
2353__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2354 enum charge_type ctype);
2355
Balbir Singhe1a1cd52008-02-07 00:14:02 -08002356int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
2357 gfp_t gfp_mask)
Balbir Singh8697d332008-02-07 00:13:59 -08002358{
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002359 int ret;
2360
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002361 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07002362 return 0;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002363 if (PageCompound(page))
2364 return 0;
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002365 /*
2366 * Corner case handling. This is called from add_to_page_cache()
2367 * in usual. But some FS (shmem) precharges this page before calling it
2368 * and call add_to_page_cache() with GFP_NOWAIT.
2369 *
2370 * For GFP_NOWAIT case, the page may be pre-charged before calling
2371 * add_to_page_cache(). (See shmem.c) check it here and avoid to call
2372 * charge twice. (It works but has to pay a bit larger cost.)
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002373 * And when the page is SwapCache, it should take swap information
2374 * into account. This is under lock_page() now.
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002375 */
2376 if (!(gfp_mask & __GFP_WAIT)) {
2377 struct page_cgroup *pc;
2378
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002379 pc = lookup_page_cgroup(page);
2380 if (!pc)
2381 return 0;
2382 lock_page_cgroup(pc);
2383 if (PageCgroupUsed(pc)) {
2384 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002385 return 0;
2386 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002387 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002388 }
2389
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002390 if (unlikely(!mm))
Balbir Singh8697d332008-02-07 00:13:59 -08002391 mm = &init_mm;
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002392
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -07002393 if (page_is_file_cache(page))
2394 return mem_cgroup_charge_common(page, mm, gfp_mask,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002395 MEM_CGROUP_CHARGE_TYPE_CACHE);
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002396
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002397 /* shmem */
2398 if (PageSwapCache(page)) {
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002399 struct mem_cgroup *mem = NULL;
2400
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002401 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2402 if (!ret)
2403 __mem_cgroup_commit_charge_swapin(page, mem,
2404 MEM_CGROUP_CHARGE_TYPE_SHMEM);
2405 } else
2406 ret = mem_cgroup_charge_common(page, mm, gfp_mask,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002407 MEM_CGROUP_CHARGE_TYPE_SHMEM);
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002408
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002409 return ret;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002410}
2411
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002412/*
2413 * While swap-in, try_charge -> commit or cancel, the page is locked.
2414 * And when try_charge() successfully returns, one refcnt to memcg without
Uwe Kleine-König21ae2952009-10-07 15:21:09 +02002415 * struct page_cgroup is acquired. This refcnt will be consumed by
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002416 * "commit()" or removed by "cancel()"
2417 */
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002418int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
2419 struct page *page,
2420 gfp_t mask, struct mem_cgroup **ptr)
2421{
2422 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002423 int ret;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002424
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002425 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002426 return 0;
2427
2428 if (!do_swap_account)
2429 goto charge_cur_mm;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002430 /*
2431 * A racing thread's fault, or swapoff, may have already updated
Hugh Dickins407f9c82009-12-14 17:59:30 -08002432 * the pte, and even removed page from swap cache: in those cases
2433 * do_swap_page()'s pte_same() test will fail; but there's also a
2434 * KSM case which does need to charge the page.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002435 */
2436 if (!PageSwapCache(page))
Hugh Dickins407f9c82009-12-14 17:59:30 -08002437 goto charge_cur_mm;
Wu Fengguange42d9d52009-12-16 12:19:59 +01002438 mem = try_get_mem_cgroup_from_page(page);
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002439 if (!mem)
2440 goto charge_cur_mm;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002441 *ptr = mem;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002442 ret = __mem_cgroup_try_charge(NULL, mask, ptr, true, PAGE_SIZE);
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002443 css_put(&mem->css);
2444 return ret;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002445charge_cur_mm:
2446 if (unlikely(!mm))
2447 mm = &init_mm;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002448 return __mem_cgroup_try_charge(mm, mask, ptr, true, PAGE_SIZE);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002449}
2450
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002451static void
2452__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2453 enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002454{
2455 struct page_cgroup *pc;
2456
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002457 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002458 return;
2459 if (!ptr)
2460 return;
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002461 cgroup_exclude_rmdir(&ptr->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002462 pc = lookup_page_cgroup(page);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08002463 mem_cgroup_lru_del_before_commit_swapcache(page);
Andrea Arcangeliec168512011-01-13 15:46:56 -08002464 __mem_cgroup_commit_charge(ptr, pc, ctype, PAGE_SIZE);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08002465 mem_cgroup_lru_add_after_commit_swapcache(page);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002466 /*
2467 * Now swap is on-memory. This means this page may be
2468 * counted both as mem and swap....double count.
KAMEZAWA Hiroyuki03f3c432009-01-07 18:08:31 -08002469 * Fix it by uncharging from memsw. Basically, this SwapCache is stable
2470 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
2471 * may call delete_from_swap_cache() before reach here.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002472 */
KAMEZAWA Hiroyuki03f3c432009-01-07 18:08:31 -08002473 if (do_swap_account && PageSwapCache(page)) {
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002474 swp_entry_t ent = {.val = page_private(page)};
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002475 unsigned short id;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002476 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002477
2478 id = swap_cgroup_record(ent, 0);
2479 rcu_read_lock();
2480 memcg = mem_cgroup_lookup(id);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002481 if (memcg) {
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002482 /*
2483 * This recorded memcg can be obsolete one. So, avoid
2484 * calling css_tryget
2485 */
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002486 if (!mem_cgroup_is_root(memcg))
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -07002487 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002488 mem_cgroup_swap_statistics(memcg, false);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002489 mem_cgroup_put(memcg);
2490 }
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002491 rcu_read_unlock();
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002492 }
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002493 /*
2494 * At swapin, we may charge account against cgroup which has no tasks.
2495 * So, rmdir()->pre_destroy() can be called while we do this charge.
2496 * In that case, we need to call pre_destroy() again. check it here.
2497 */
2498 cgroup_release_and_wakeup_rmdir(&ptr->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002499}
2500
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002501void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
2502{
2503 __mem_cgroup_commit_charge_swapin(page, ptr,
2504 MEM_CGROUP_CHARGE_TYPE_MAPPED);
2505}
2506
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002507void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
2508{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002509 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002510 return;
2511 if (!mem)
2512 return;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002513 mem_cgroup_cancel_charge(mem, PAGE_SIZE);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002514}
2515
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002516static void
Andrea Arcangeliec168512011-01-13 15:46:56 -08002517__do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype,
2518 int page_size)
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002519{
2520 struct memcg_batch_info *batch = NULL;
2521 bool uncharge_memsw = true;
2522 /* If swapout, usage of swap doesn't decrease */
2523 if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
2524 uncharge_memsw = false;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002525
2526 batch = &current->memcg_batch;
2527 /*
2528 * In usual, we do css_get() when we remember memcg pointer.
2529 * But in this case, we keep res->usage until end of a series of
2530 * uncharges. Then, it's ok to ignore memcg's refcnt.
2531 */
2532 if (!batch->memcg)
2533 batch->memcg = mem;
2534 /*
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002535 * do_batch > 0 when unmapping pages or inode invalidate/truncate.
2536 * In those cases, all pages freed continously can be expected to be in
2537 * the same cgroup and we have chance to coalesce uncharges.
2538 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
2539 * because we want to do uncharge as soon as possible.
2540 */
2541
2542 if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
2543 goto direct_uncharge;
2544
Andrea Arcangeliec168512011-01-13 15:46:56 -08002545 if (page_size != PAGE_SIZE)
2546 goto direct_uncharge;
2547
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002548 /*
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002549 * In typical case, batch->memcg == mem. This means we can
2550 * merge a series of uncharges to an uncharge of res_counter.
2551 * If not, we uncharge res_counter ony by one.
2552 */
2553 if (batch->memcg != mem)
2554 goto direct_uncharge;
2555 /* remember freed charge and uncharge it later */
2556 batch->bytes += PAGE_SIZE;
2557 if (uncharge_memsw)
2558 batch->memsw_bytes += PAGE_SIZE;
2559 return;
2560direct_uncharge:
Andrea Arcangeliec168512011-01-13 15:46:56 -08002561 res_counter_uncharge(&mem->res, page_size);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002562 if (uncharge_memsw)
Andrea Arcangeliec168512011-01-13 15:46:56 -08002563 res_counter_uncharge(&mem->memsw, page_size);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002564 if (unlikely(batch->memcg != mem))
2565 memcg_oom_recover(mem);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002566 return;
2567}
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002568
Balbir Singh8697d332008-02-07 00:13:59 -08002569/*
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002570 * uncharge if !page_mapped(page)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002571 */
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002572static struct mem_cgroup *
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002573__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002574{
Daisuke Nishimura152c9cc2011-01-13 15:46:56 -08002575 int count;
Hugh Dickins82895462008-03-04 14:29:08 -08002576 struct page_cgroup *pc;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002577 struct mem_cgroup *mem = NULL;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002578 int page_size = PAGE_SIZE;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002579
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002580 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002581 return NULL;
Balbir Singh40779602008-04-04 14:29:59 -07002582
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002583 if (PageSwapCache(page))
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002584 return NULL;
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002585
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08002586 if (PageTransHuge(page)) {
Andrea Arcangeliec168512011-01-13 15:46:56 -08002587 page_size <<= compound_order(page);
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08002588 VM_BUG_ON(!PageTransHuge(page));
2589 }
Andrea Arcangeliec168512011-01-13 15:46:56 -08002590
Daisuke Nishimura152c9cc2011-01-13 15:46:56 -08002591 count = page_size >> PAGE_SHIFT;
Balbir Singh8697d332008-02-07 00:13:59 -08002592 /*
Balbir Singh3c541e12008-02-07 00:14:41 -08002593 * Check if our page_cgroup is valid
Balbir Singh8697d332008-02-07 00:13:59 -08002594 */
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002595 pc = lookup_page_cgroup(page);
2596 if (unlikely(!pc || !PageCgroupUsed(pc)))
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002597 return NULL;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002598
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002599 lock_page_cgroup(pc);
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002600
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002601 mem = pc->mem_cgroup;
2602
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002603 if (!PageCgroupUsed(pc))
2604 goto unlock_out;
2605
2606 switch (ctype) {
2607 case MEM_CGROUP_CHARGE_TYPE_MAPPED:
KAMEZAWA Hiroyuki8a9478ca2009-06-17 16:27:17 -07002608 case MEM_CGROUP_CHARGE_TYPE_DROP:
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002609 /* See mem_cgroup_prepare_migration() */
2610 if (page_mapped(page) || PageCgroupMigration(pc))
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002611 goto unlock_out;
2612 break;
2613 case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
2614 if (!PageAnon(page)) { /* Shared memory */
2615 if (page->mapping && !page_is_file_cache(page))
2616 goto unlock_out;
2617 } else if (page_mapped(page)) /* Anon */
2618 goto unlock_out;
2619 break;
2620 default:
2621 break;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002622 }
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002623
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002624 mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), -count);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07002625
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002626 ClearPageCgroupUsed(pc);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08002627 /*
2628 * pc->mem_cgroup is not cleared here. It will be accessed when it's
2629 * freed from LRU. This is safe because uncharged page is expected not
2630 * to be reused (freed soon). Exception is SwapCache, it's handled by
2631 * special functions.
2632 */
Hugh Dickinsb9c565d2008-03-04 14:29:11 -08002633
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002634 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002635 /*
2636 * even after unlock, we have mem->res.usage here and this memcg
2637 * will never be freed.
2638 */
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08002639 memcg_check_events(mem, page);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002640 if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
2641 mem_cgroup_swap_statistics(mem, true);
2642 mem_cgroup_get(mem);
2643 }
2644 if (!mem_cgroup_is_root(mem))
Andrea Arcangeliec168512011-01-13 15:46:56 -08002645 __do_uncharge(mem, ctype, page_size);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08002646
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002647 return mem;
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002648
2649unlock_out:
2650 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002651 return NULL;
Balbir Singh3c541e12008-02-07 00:14:41 -08002652}
2653
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002654void mem_cgroup_uncharge_page(struct page *page)
2655{
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002656 /* early check. */
2657 if (page_mapped(page))
2658 return;
2659 if (page->mapping && !PageAnon(page))
2660 return;
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002661 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
2662}
2663
2664void mem_cgroup_uncharge_cache_page(struct page *page)
2665{
2666 VM_BUG_ON(page_mapped(page));
KAMEZAWA Hiroyukib7abea92008-10-18 20:28:09 -07002667 VM_BUG_ON(page->mapping);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002668 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
2669}
2670
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002671/*
2672 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
2673 * In that cases, pages are freed continuously and we can expect pages
2674 * are in the same memcg. All these calls itself limits the number of
2675 * pages freed at once, then uncharge_start/end() is called properly.
2676 * This may be called prural(2) times in a context,
2677 */
2678
2679void mem_cgroup_uncharge_start(void)
2680{
2681 current->memcg_batch.do_batch++;
2682 /* We can do nest. */
2683 if (current->memcg_batch.do_batch == 1) {
2684 current->memcg_batch.memcg = NULL;
2685 current->memcg_batch.bytes = 0;
2686 current->memcg_batch.memsw_bytes = 0;
2687 }
2688}
2689
2690void mem_cgroup_uncharge_end(void)
2691{
2692 struct memcg_batch_info *batch = &current->memcg_batch;
2693
2694 if (!batch->do_batch)
2695 return;
2696
2697 batch->do_batch--;
2698 if (batch->do_batch) /* If stacked, do nothing. */
2699 return;
2700
2701 if (!batch->memcg)
2702 return;
2703 /*
2704 * This "batch->memcg" is valid without any css_get/put etc...
2705 * bacause we hide charges behind us.
2706 */
2707 if (batch->bytes)
2708 res_counter_uncharge(&batch->memcg->res, batch->bytes);
2709 if (batch->memsw_bytes)
2710 res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002711 memcg_oom_recover(batch->memcg);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002712 /* forget this pointer (for sanity check) */
2713 batch->memcg = NULL;
2714}
2715
Daisuke Nishimurae767e052009-05-28 14:34:28 -07002716#ifdef CONFIG_SWAP
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002717/*
Daisuke Nishimurae767e052009-05-28 14:34:28 -07002718 * called after __delete_from_swap_cache() and drop "page" account.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002719 * memcg information is recorded to swap_cgroup of "ent"
2720 */
KAMEZAWA Hiroyuki8a9478ca2009-06-17 16:27:17 -07002721void
2722mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002723{
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002724 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki8a9478ca2009-06-17 16:27:17 -07002725 int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002726
KAMEZAWA Hiroyuki8a9478ca2009-06-17 16:27:17 -07002727 if (!swapout) /* this was a swap cache but the swap is unused ! */
2728 ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
2729
2730 memcg = __mem_cgroup_uncharge_common(page, ctype);
2731
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002732 /*
2733 * record memcg information, if swapout && memcg != NULL,
2734 * mem_cgroup_get() was called in uncharge().
2735 */
2736 if (do_swap_account && swapout && memcg)
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002737 swap_cgroup_record(ent, css_id(&memcg->css));
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002738}
Daisuke Nishimurae767e052009-05-28 14:34:28 -07002739#endif
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002740
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002741#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
2742/*
2743 * called from swap_entry_free(). remove record in swap_cgroup and
2744 * uncharge "memsw" account.
2745 */
2746void mem_cgroup_uncharge_swap(swp_entry_t ent)
2747{
2748 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002749 unsigned short id;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002750
2751 if (!do_swap_account)
2752 return;
2753
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002754 id = swap_cgroup_record(ent, 0);
2755 rcu_read_lock();
2756 memcg = mem_cgroup_lookup(id);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002757 if (memcg) {
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002758 /*
2759 * We uncharge this because swap is freed.
2760 * This memcg can be obsolete one. We avoid calling css_tryget
2761 */
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002762 if (!mem_cgroup_is_root(memcg))
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -07002763 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002764 mem_cgroup_swap_statistics(memcg, false);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002765 mem_cgroup_put(memcg);
2766 }
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002767 rcu_read_unlock();
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002768}
Daisuke Nishimura02491442010-03-10 15:22:17 -08002769
2770/**
2771 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2772 * @entry: swap entry to be moved
2773 * @from: mem_cgroup which the entry is moved from
2774 * @to: mem_cgroup which the entry is moved to
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002775 * @need_fixup: whether we should fixup res_counters and refcounts.
Daisuke Nishimura02491442010-03-10 15:22:17 -08002776 *
2777 * It succeeds only when the swap_cgroup's record for this entry is the same
2778 * as the mem_cgroup's id of @from.
2779 *
2780 * Returns 0 on success, -EINVAL on failure.
2781 *
2782 * The caller must have charged to @to, IOW, called res_counter_charge() about
2783 * both res and memsw, and called css_get().
2784 */
2785static int mem_cgroup_move_swap_account(swp_entry_t entry,
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002786 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
Daisuke Nishimura02491442010-03-10 15:22:17 -08002787{
2788 unsigned short old_id, new_id;
2789
2790 old_id = css_id(&from->css);
2791 new_id = css_id(&to->css);
2792
2793 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08002794 mem_cgroup_swap_statistics(from, false);
Daisuke Nishimura02491442010-03-10 15:22:17 -08002795 mem_cgroup_swap_statistics(to, true);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002796 /*
2797 * This function is only called from task migration context now.
2798 * It postpones res_counter and refcount handling till the end
2799 * of task migration(mem_cgroup_clear_mc()) for performance
2800 * improvement. But we cannot postpone mem_cgroup_get(to)
2801 * because if the process that has been moved to @to does
2802 * swap-in, the refcount of @to might be decreased to 0.
2803 */
Daisuke Nishimura02491442010-03-10 15:22:17 -08002804 mem_cgroup_get(to);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002805 if (need_fixup) {
2806 if (!mem_cgroup_is_root(from))
2807 res_counter_uncharge(&from->memsw, PAGE_SIZE);
2808 mem_cgroup_put(from);
2809 /*
2810 * we charged both to->res and to->memsw, so we should
2811 * uncharge to->res.
2812 */
2813 if (!mem_cgroup_is_root(to))
2814 res_counter_uncharge(&to->res, PAGE_SIZE);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002815 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08002816 return 0;
2817 }
2818 return -EINVAL;
2819}
2820#else
2821static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002822 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
Daisuke Nishimura02491442010-03-10 15:22:17 -08002823{
2824 return -EINVAL;
2825}
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002826#endif
2827
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08002828/*
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002829 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
2830 * page belongs to.
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08002831 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002832int mem_cgroup_prepare_migration(struct page *page,
2833 struct page *newpage, struct mem_cgroup **ptr)
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08002834{
2835 struct page_cgroup *pc;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002836 struct mem_cgroup *mem = NULL;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002837 enum charge_type ctype;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002838 int ret = 0;
Hugh Dickins8869b8f2008-03-04 14:29:09 -08002839
Andrea Arcangeliec168512011-01-13 15:46:56 -08002840 VM_BUG_ON(PageTransHuge(page));
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002841 if (mem_cgroup_disabled())
Balbir Singh40779602008-04-04 14:29:59 -07002842 return 0;
2843
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002844 pc = lookup_page_cgroup(page);
2845 lock_page_cgroup(pc);
2846 if (PageCgroupUsed(pc)) {
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002847 mem = pc->mem_cgroup;
2848 css_get(&mem->css);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002849 /*
2850 * At migrating an anonymous page, its mapcount goes down
2851 * to 0 and uncharge() will be called. But, even if it's fully
2852 * unmapped, migration may fail and this page has to be
2853 * charged again. We set MIGRATION flag here and delay uncharge
2854 * until end_migration() is called
2855 *
2856 * Corner Case Thinking
2857 * A)
2858 * When the old page was mapped as Anon and it's unmap-and-freed
2859 * while migration was ongoing.
2860 * If unmap finds the old page, uncharge() of it will be delayed
2861 * until end_migration(). If unmap finds a new page, it's
2862 * uncharged when it make mapcount to be 1->0. If unmap code
2863 * finds swap_migration_entry, the new page will not be mapped
2864 * and end_migration() will find it(mapcount==0).
2865 *
2866 * B)
2867 * When the old page was mapped but migraion fails, the kernel
2868 * remaps it. A charge for it is kept by MIGRATION flag even
2869 * if mapcount goes down to 0. We can do remap successfully
2870 * without charging it again.
2871 *
2872 * C)
2873 * The "old" page is under lock_page() until the end of
2874 * migration, so, the old page itself will not be swapped-out.
2875 * If the new page is swapped out before end_migraton, our
2876 * hook to usual swap-out path will catch the event.
2877 */
2878 if (PageAnon(page))
2879 SetPageCgroupMigration(pc);
Hugh Dickinsb9c565d2008-03-04 14:29:11 -08002880 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002881 unlock_page_cgroup(pc);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002882 /*
2883 * If the page is not charged at this point,
2884 * we return here.
2885 */
2886 if (!mem)
2887 return 0;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002888
Andrea Arcangeli93d5c9b2010-04-23 13:17:39 -04002889 *ptr = mem;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002890 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, ptr, false, PAGE_SIZE);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002891 css_put(&mem->css);/* drop extra refcnt */
2892 if (ret || *ptr == NULL) {
2893 if (PageAnon(page)) {
2894 lock_page_cgroup(pc);
2895 ClearPageCgroupMigration(pc);
2896 unlock_page_cgroup(pc);
2897 /*
2898 * The old page may be fully unmapped while we kept it.
2899 */
2900 mem_cgroup_uncharge_page(page);
2901 }
2902 return -ENOMEM;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002903 }
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002904 /*
2905 * We charge new page before it's used/mapped. So, even if unlock_page()
2906 * is called before end_migration, we can catch all events on this new
2907 * page. In the case new page is migrated but not remapped, new page's
2908 * mapcount will be finally 0 and we call uncharge in end_migration().
2909 */
2910 pc = lookup_page_cgroup(newpage);
2911 if (PageAnon(page))
2912 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
2913 else if (page_is_file_cache(page))
2914 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
2915 else
2916 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002917 __mem_cgroup_commit_charge(mem, pc, ctype, PAGE_SIZE);
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002918 return ret;
2919}
Hugh Dickinsfb59e9f2008-03-04 14:29:16 -08002920
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002921/* remove redundant charge if migration failed*/
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002922void mem_cgroup_end_migration(struct mem_cgroup *mem,
Daisuke Nishimura50de1dd2011-01-13 15:47:43 -08002923 struct page *oldpage, struct page *newpage, bool migration_ok)
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002924{
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002925 struct page *used, *unused;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002926 struct page_cgroup *pc;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002927
2928 if (!mem)
2929 return;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002930 /* blocks rmdir() */
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002931 cgroup_exclude_rmdir(&mem->css);
Daisuke Nishimura50de1dd2011-01-13 15:47:43 -08002932 if (!migration_ok) {
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002933 used = oldpage;
2934 unused = newpage;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002935 } else {
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002936 used = newpage;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002937 unused = oldpage;
2938 }
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002939 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002940 * We disallowed uncharge of pages under migration because mapcount
2941 * of the page goes down to zero, temporarly.
2942 * Clear the flag and check the page should be charged.
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002943 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002944 pc = lookup_page_cgroup(oldpage);
2945 lock_page_cgroup(pc);
2946 ClearPageCgroupMigration(pc);
2947 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002948
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002949 __mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE);
2950
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002951 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002952 * If a page is a file cache, radix-tree replacement is very atomic
2953 * and we can skip this check. When it was an Anon page, its mapcount
2954 * goes down to 0. But because we added MIGRATION flage, it's not
2955 * uncharged yet. There are several case but page->mapcount check
2956 * and USED bit check in mem_cgroup_uncharge_page() will do enough
2957 * check. (see prepare_charge() also)
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002958 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002959 if (PageAnon(used))
2960 mem_cgroup_uncharge_page(used);
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002961 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002962 * At migration, we may charge account against cgroup which has no
2963 * tasks.
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002964 * So, rmdir()->pre_destroy() can be called while we do this charge.
2965 * In that case, we need to call pre_destroy() again. check it here.
2966 */
2967 cgroup_release_and_wakeup_rmdir(&mem->css);
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08002968}
Pavel Emelianov78fb7462008-02-07 00:13:51 -08002969
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08002970/*
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002971 * A call to try to shrink memory usage on charge failure at shmem's swapin.
2972 * Calling hierarchical_reclaim is not enough because we should update
2973 * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
2974 * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
2975 * not from the memcg which this page would be charged to.
2976 * try_charge_swapin does all of these works properly.
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002977 */
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002978int mem_cgroup_shmem_charge_fallback(struct page *page,
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002979 struct mm_struct *mm,
2980 gfp_t gfp_mask)
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002981{
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002982 struct mem_cgroup *mem = NULL;
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002983 int ret;
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002984
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002985 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07002986 return 0;
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002987
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002988 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2989 if (!ret)
2990 mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002991
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002992 return ret;
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002993}
2994
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002995static DEFINE_MUTEX(set_limit_mutex);
2996
KOSAKI Motohirod38d2a72009-01-06 14:39:44 -08002997static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002998 unsigned long long val)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002999{
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003000 int retry_count;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003001 u64 memswlimit, memlimit;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003002 int ret = 0;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003003 int children = mem_cgroup_count_children(memcg);
3004 u64 curusage, oldusage;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003005 int enlarge;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003006
3007 /*
3008 * For keeping hierarchical_reclaim simple, how long we should retry
3009 * is depends on callers. We set our retry-count to be function
3010 * of # of children which we should visit in this loop.
3011 */
3012 retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
3013
3014 oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003015
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003016 enlarge = 0;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003017 while (retry_count) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003018 if (signal_pending(current)) {
3019 ret = -EINTR;
3020 break;
3021 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003022 /*
3023 * Rather than hide all in some function, I do this in
3024 * open coded manner. You see what this really does.
3025 * We have to guarantee mem->res.limit < mem->memsw.limit.
3026 */
3027 mutex_lock(&set_limit_mutex);
3028 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3029 if (memswlimit < val) {
3030 ret = -EINVAL;
3031 mutex_unlock(&set_limit_mutex);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003032 break;
3033 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003034
3035 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3036 if (memlimit < val)
3037 enlarge = 1;
3038
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003039 ret = res_counter_set_limit(&memcg->res, val);
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07003040 if (!ret) {
3041 if (memswlimit == val)
3042 memcg->memsw_is_minimum = true;
3043 else
3044 memcg->memsw_is_minimum = false;
3045 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003046 mutex_unlock(&set_limit_mutex);
3047
3048 if (!ret)
3049 break;
3050
Bob Liuaa20d482009-12-15 16:47:14 -08003051 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
Balbir Singh4e416952009-09-23 15:56:39 -07003052 MEM_CGROUP_RECLAIM_SHRINK);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003053 curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
3054 /* Usage is reduced ? */
3055 if (curusage >= oldusage)
3056 retry_count--;
3057 else
3058 oldusage = curusage;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003059 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003060 if (!ret && enlarge)
3061 memcg_oom_recover(memcg);
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08003062
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003063 return ret;
3064}
3065
Li Zefan338c8432009-06-17 16:27:15 -07003066static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
3067 unsigned long long val)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003068{
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003069 int retry_count;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003070 u64 memlimit, memswlimit, oldusage, curusage;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003071 int children = mem_cgroup_count_children(memcg);
3072 int ret = -EBUSY;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003073 int enlarge = 0;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003074
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003075 /* see mem_cgroup_resize_res_limit */
3076 retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
3077 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003078 while (retry_count) {
3079 if (signal_pending(current)) {
3080 ret = -EINTR;
3081 break;
3082 }
3083 /*
3084 * Rather than hide all in some function, I do this in
3085 * open coded manner. You see what this really does.
3086 * We have to guarantee mem->res.limit < mem->memsw.limit.
3087 */
3088 mutex_lock(&set_limit_mutex);
3089 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3090 if (memlimit > val) {
3091 ret = -EINVAL;
3092 mutex_unlock(&set_limit_mutex);
3093 break;
3094 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003095 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3096 if (memswlimit < val)
3097 enlarge = 1;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003098 ret = res_counter_set_limit(&memcg->memsw, val);
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07003099 if (!ret) {
3100 if (memlimit == val)
3101 memcg->memsw_is_minimum = true;
3102 else
3103 memcg->memsw_is_minimum = false;
3104 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003105 mutex_unlock(&set_limit_mutex);
3106
3107 if (!ret)
3108 break;
3109
Balbir Singh4e416952009-09-23 15:56:39 -07003110 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
Balbir Singh75822b42009-09-23 15:56:38 -07003111 MEM_CGROUP_RECLAIM_NOSWAP |
3112 MEM_CGROUP_RECLAIM_SHRINK);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003113 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003114 /* Usage is reduced ? */
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003115 if (curusage >= oldusage)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003116 retry_count--;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003117 else
3118 oldusage = curusage;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003119 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003120 if (!ret && enlarge)
3121 memcg_oom_recover(memcg);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003122 return ret;
3123}
3124
Balbir Singh4e416952009-09-23 15:56:39 -07003125unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
KOSAKI Motohiro00918b62010-08-10 18:03:05 -07003126 gfp_t gfp_mask)
Balbir Singh4e416952009-09-23 15:56:39 -07003127{
3128 unsigned long nr_reclaimed = 0;
3129 struct mem_cgroup_per_zone *mz, *next_mz = NULL;
3130 unsigned long reclaimed;
3131 int loop = 0;
3132 struct mem_cgroup_tree_per_zone *mctz;
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07003133 unsigned long long excess;
Balbir Singh4e416952009-09-23 15:56:39 -07003134
3135 if (order > 0)
3136 return 0;
3137
KOSAKI Motohiro00918b62010-08-10 18:03:05 -07003138 mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
Balbir Singh4e416952009-09-23 15:56:39 -07003139 /*
3140 * This loop can run a while, specially if mem_cgroup's continuously
3141 * keep exceeding their soft limit and putting the system under
3142 * pressure
3143 */
3144 do {
3145 if (next_mz)
3146 mz = next_mz;
3147 else
3148 mz = mem_cgroup_largest_soft_limit_node(mctz);
3149 if (!mz)
3150 break;
3151
3152 reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
3153 gfp_mask,
3154 MEM_CGROUP_RECLAIM_SOFT);
3155 nr_reclaimed += reclaimed;
3156 spin_lock(&mctz->lock);
3157
3158 /*
3159 * If we failed to reclaim anything from this memory cgroup
3160 * it is time to move on to the next cgroup
3161 */
3162 next_mz = NULL;
3163 if (!reclaimed) {
3164 do {
3165 /*
3166 * Loop until we find yet another one.
3167 *
3168 * By the time we get the soft_limit lock
3169 * again, someone might have aded the
3170 * group back on the RB tree. Iterate to
3171 * make sure we get a different mem.
3172 * mem_cgroup_largest_soft_limit_node returns
3173 * NULL if no other cgroup is present on
3174 * the tree
3175 */
3176 next_mz =
3177 __mem_cgroup_largest_soft_limit_node(mctz);
3178 if (next_mz == mz) {
3179 css_put(&next_mz->mem->css);
3180 next_mz = NULL;
3181 } else /* next_mz == NULL or other memcg */
3182 break;
3183 } while (1);
3184 }
Balbir Singh4e416952009-09-23 15:56:39 -07003185 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07003186 excess = res_counter_soft_limit_excess(&mz->mem->res);
Balbir Singh4e416952009-09-23 15:56:39 -07003187 /*
3188 * One school of thought says that we should not add
3189 * back the node to the tree if reclaim returns 0.
3190 * But our reclaim could return 0, simply because due
3191 * to priority we are exposing a smaller subset of
3192 * memory to reclaim from. Consider this as a longer
3193 * term TODO.
3194 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07003195 /* If excess == 0, no tree ops */
3196 __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
Balbir Singh4e416952009-09-23 15:56:39 -07003197 spin_unlock(&mctz->lock);
3198 css_put(&mz->mem->css);
3199 loop++;
3200 /*
3201 * Could not reclaim anything and there are no more
3202 * mem cgroups to try or we seem to be looping without
3203 * reclaiming anything.
3204 */
3205 if (!nr_reclaimed &&
3206 (next_mz == NULL ||
3207 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
3208 break;
3209 } while (!nr_reclaimed);
3210 if (next_mz)
3211 css_put(&next_mz->mem->css);
3212 return nr_reclaimed;
3213}
3214
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07003215/*
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003216 * This routine traverse page_cgroup in given list and drop them all.
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003217 * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
3218 */
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003219static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003220 int node, int zid, enum lru_list lru)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003221{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003222 struct zone *zone;
3223 struct mem_cgroup_per_zone *mz;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003224 struct page_cgroup *pc, *busy;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003225 unsigned long flags, loop;
KAMEZAWA Hiroyuki072c56c12008-02-07 00:14:39 -08003226 struct list_head *list;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003227 int ret = 0;
KAMEZAWA Hiroyuki072c56c12008-02-07 00:14:39 -08003228
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003229 zone = &NODE_DATA(node)->node_zones[zid];
3230 mz = mem_cgroup_zoneinfo(mem, node, zid);
Christoph Lameterb69408e2008-10-18 20:26:14 -07003231 list = &mz->lists[lru];
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003232
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003233 loop = MEM_CGROUP_ZSTAT(mz, lru);
3234 /* give some margin against EBUSY etc...*/
3235 loop += 256;
3236 busy = NULL;
3237 while (loop--) {
3238 ret = 0;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003239 spin_lock_irqsave(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003240 if (list_empty(list)) {
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003241 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003242 break;
3243 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003244 pc = list_entry(list->prev, struct page_cgroup, lru);
3245 if (busy == pc) {
3246 list_move(&pc->lru, list);
Thiago Farina648bcc72010-03-05 13:42:04 -08003247 busy = NULL;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003248 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003249 continue;
3250 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003251 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003252
KAMEZAWA Hiroyuki2c26fdd2009-01-07 18:08:10 -08003253 ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003254 if (ret == -ENOMEM)
3255 break;
3256
3257 if (ret == -EBUSY || ret == -EINVAL) {
3258 /* found lock contention or "pc" is obsolete. */
3259 busy = pc;
3260 cond_resched();
3261 } else
3262 busy = NULL;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003263 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003264
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003265 if (!ret && !list_empty(list))
3266 return -EBUSY;
3267 return ret;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003268}
3269
3270/*
3271 * make mem_cgroup's charge to be 0 if there is no task.
3272 * This enables deleting this mem_cgroup.
3273 */
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003274static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003275{
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003276 int ret;
3277 int node, zid, shrink;
3278 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003279 struct cgroup *cgrp = mem->css.cgroup;
Hugh Dickins8869b8f2008-03-04 14:29:09 -08003280
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003281 css_get(&mem->css);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003282
3283 shrink = 0;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003284 /* should free all ? */
3285 if (free_all)
3286 goto try_to_free;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003287move_account:
Daisuke Nishimurafce66472010-01-15 17:01:30 -08003288 do {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003289 ret = -EBUSY;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003290 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003291 goto out;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003292 ret = -EINTR;
3293 if (signal_pending(current))
3294 goto out;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003295 /* This is for making all *used* pages to be on LRU. */
3296 lru_add_drain_all();
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08003297 drain_all_stock_sync();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003298 ret = 0;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07003299 mem_cgroup_start_move(mem);
KAMEZAWA Hiroyuki299b4ea2009-01-29 14:25:17 -08003300 for_each_node_state(node, N_HIGH_MEMORY) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003301 for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
Christoph Lameterb69408e2008-10-18 20:26:14 -07003302 enum lru_list l;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003303 for_each_lru(l) {
3304 ret = mem_cgroup_force_empty_list(mem,
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003305 node, zid, l);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003306 if (ret)
3307 break;
3308 }
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08003309 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003310 if (ret)
3311 break;
3312 }
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07003313 mem_cgroup_end_move(mem);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003314 memcg_oom_recover(mem);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003315 /* it seems parent cgroup doesn't have enough mem */
3316 if (ret == -ENOMEM)
3317 goto try_to_free;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003318 cond_resched();
Daisuke Nishimurafce66472010-01-15 17:01:30 -08003319 /* "ret" should also be checked to ensure all lists are empty. */
3320 } while (mem->res.usage > 0 || ret);
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003321out:
3322 css_put(&mem->css);
3323 return ret;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003324
3325try_to_free:
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003326 /* returns EBUSY if there is a task or if we come here twice. */
3327 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003328 ret = -EBUSY;
3329 goto out;
3330 }
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003331 /* we call try-to-free pages for make this cgroup empty */
3332 lru_add_drain_all();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003333 /* try to free all pages in this cgroup */
3334 shrink = 1;
3335 while (nr_retries && mem->res.usage > 0) {
3336 int progress;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003337
3338 if (signal_pending(current)) {
3339 ret = -EINTR;
3340 goto out;
3341 }
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003342 progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
3343 false, get_swappiness(mem));
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003344 if (!progress) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003345 nr_retries--;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003346 /* maybe some writeback is necessary */
Jens Axboe8aa7e842009-07-09 14:52:32 +02003347 congestion_wait(BLK_RW_ASYNC, HZ/10);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003348 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003349
3350 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003351 lru_add_drain();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003352 /* try move_account...there may be some *locked* pages. */
Daisuke Nishimurafce66472010-01-15 17:01:30 -08003353 goto move_account;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003354}
3355
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003356int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
3357{
3358 return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
3359}
3360
3361
Balbir Singh18f59ea2009-01-07 18:08:07 -08003362static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
3363{
3364 return mem_cgroup_from_cont(cont)->use_hierarchy;
3365}
3366
3367static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
3368 u64 val)
3369{
3370 int retval = 0;
3371 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
3372 struct cgroup *parent = cont->parent;
3373 struct mem_cgroup *parent_mem = NULL;
3374
3375 if (parent)
3376 parent_mem = mem_cgroup_from_cont(parent);
3377
3378 cgroup_lock();
3379 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02003380 * If parent's use_hierarchy is set, we can't make any modifications
Balbir Singh18f59ea2009-01-07 18:08:07 -08003381 * in the child subtrees. If it is unset, then the change can
3382 * occur, provided the current cgroup has no children.
3383 *
3384 * For the root cgroup, parent_mem is NULL, we allow value to be
3385 * set if there are no children.
3386 */
3387 if ((!parent_mem || !parent_mem->use_hierarchy) &&
3388 (val == 1 || val == 0)) {
3389 if (list_empty(&cont->children))
3390 mem->use_hierarchy = val;
3391 else
3392 retval = -EBUSY;
3393 } else
3394 retval = -EINVAL;
3395 cgroup_unlock();
3396
3397 return retval;
3398}
3399
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003400
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003401static u64 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
3402 enum mem_cgroup_stat_index idx)
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003403{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003404 struct mem_cgroup *iter;
3405 s64 val = 0;
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003406
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003407 /* each per cpu's value can be minus.Then, use s64 */
3408 for_each_mem_cgroup_tree(iter, mem)
3409 val += mem_cgroup_read_stat(iter, idx);
3410
3411 if (val < 0) /* race ? */
3412 val = 0;
3413 return val;
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003414}
3415
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003416static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
3417{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003418 u64 val;
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003419
3420 if (!mem_cgroup_is_root(mem)) {
3421 if (!swap)
3422 return res_counter_read_u64(&mem->res, RES_USAGE);
3423 else
3424 return res_counter_read_u64(&mem->memsw, RES_USAGE);
3425 }
3426
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003427 val = mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE);
3428 val += mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003429
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003430 if (swap)
3431 val += mem_cgroup_get_recursive_idx_stat(mem,
3432 MEM_CGROUP_STAT_SWAPOUT);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003433
3434 return val << PAGE_SHIFT;
3435}
3436
Paul Menage2c3daa72008-04-29 00:59:58 -07003437static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003438{
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003439 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003440 u64 val;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003441 int type, name;
3442
3443 type = MEMFILE_TYPE(cft->private);
3444 name = MEMFILE_ATTR(cft->private);
3445 switch (type) {
3446 case _MEM:
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003447 if (name == RES_USAGE)
3448 val = mem_cgroup_usage(mem, false);
3449 else
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003450 val = res_counter_read_u64(&mem->res, name);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003451 break;
3452 case _MEMSWAP:
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003453 if (name == RES_USAGE)
3454 val = mem_cgroup_usage(mem, true);
3455 else
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003456 val = res_counter_read_u64(&mem->memsw, name);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003457 break;
3458 default:
3459 BUG();
3460 break;
3461 }
3462 return val;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003463}
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003464/*
3465 * The user of this function is...
3466 * RES_LIMIT.
3467 */
Paul Menage856c13a2008-07-25 01:47:04 -07003468static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
3469 const char *buffer)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003470{
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003471 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003472 int type, name;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003473 unsigned long long val;
3474 int ret;
3475
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003476 type = MEMFILE_TYPE(cft->private);
3477 name = MEMFILE_ATTR(cft->private);
3478 switch (name) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003479 case RES_LIMIT:
Balbir Singh4b3bde42009-09-23 15:56:32 -07003480 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3481 ret = -EINVAL;
3482 break;
3483 }
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003484 /* This function does all necessary parse...reuse it */
3485 ret = res_counter_memparse_write_strategy(buffer, &val);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003486 if (ret)
3487 break;
3488 if (type == _MEM)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003489 ret = mem_cgroup_resize_limit(memcg, val);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003490 else
3491 ret = mem_cgroup_resize_memsw_limit(memcg, val);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003492 break;
Balbir Singh296c81d2009-09-23 15:56:36 -07003493 case RES_SOFT_LIMIT:
3494 ret = res_counter_memparse_write_strategy(buffer, &val);
3495 if (ret)
3496 break;
3497 /*
3498 * For memsw, soft limits are hard to implement in terms
3499 * of semantics, for now, we support soft limits for
3500 * control without swap
3501 */
3502 if (type == _MEM)
3503 ret = res_counter_set_soft_limit(&memcg->res, val);
3504 else
3505 ret = -EINVAL;
3506 break;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003507 default:
3508 ret = -EINVAL; /* should be BUG() ? */
3509 break;
3510 }
3511 return ret;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003512}
3513
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08003514static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
3515 unsigned long long *mem_limit, unsigned long long *memsw_limit)
3516{
3517 struct cgroup *cgroup;
3518 unsigned long long min_limit, min_memsw_limit, tmp;
3519
3520 min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3521 min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3522 cgroup = memcg->css.cgroup;
3523 if (!memcg->use_hierarchy)
3524 goto out;
3525
3526 while (cgroup->parent) {
3527 cgroup = cgroup->parent;
3528 memcg = mem_cgroup_from_cont(cgroup);
3529 if (!memcg->use_hierarchy)
3530 break;
3531 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
3532 min_limit = min(min_limit, tmp);
3533 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3534 min_memsw_limit = min(min_memsw_limit, tmp);
3535 }
3536out:
3537 *mem_limit = min_limit;
3538 *memsw_limit = min_memsw_limit;
3539 return;
3540}
3541
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003542static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003543{
3544 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003545 int type, name;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003546
3547 mem = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003548 type = MEMFILE_TYPE(event);
3549 name = MEMFILE_ATTR(event);
3550 switch (name) {
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003551 case RES_MAX_USAGE:
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003552 if (type == _MEM)
3553 res_counter_reset_max(&mem->res);
3554 else
3555 res_counter_reset_max(&mem->memsw);
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003556 break;
3557 case RES_FAILCNT:
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003558 if (type == _MEM)
3559 res_counter_reset_failcnt(&mem->res);
3560 else
3561 res_counter_reset_failcnt(&mem->memsw);
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003562 break;
3563 }
Balbir Singhf64c3f52009-09-23 15:56:37 -07003564
Pavel Emelyanov85cc59d2008-04-29 01:00:20 -07003565 return 0;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003566}
3567
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003568static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
3569 struct cftype *cft)
3570{
3571 return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
3572}
3573
Daisuke Nishimura02491442010-03-10 15:22:17 -08003574#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003575static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
3576 struct cftype *cft, u64 val)
3577{
3578 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3579
3580 if (val >= (1 << NR_MOVE_TYPE))
3581 return -EINVAL;
3582 /*
3583 * We check this value several times in both in can_attach() and
3584 * attach(), so we need cgroup lock to prevent this value from being
3585 * inconsistent.
3586 */
3587 cgroup_lock();
3588 mem->move_charge_at_immigrate = val;
3589 cgroup_unlock();
3590
3591 return 0;
3592}
Daisuke Nishimura02491442010-03-10 15:22:17 -08003593#else
3594static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
3595 struct cftype *cft, u64 val)
3596{
3597 return -ENOSYS;
3598}
3599#endif
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003600
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003601
3602/* For read statistics */
3603enum {
3604 MCS_CACHE,
3605 MCS_RSS,
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -08003606 MCS_FILE_MAPPED,
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003607 MCS_PGPGIN,
3608 MCS_PGPGOUT,
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003609 MCS_SWAP,
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003610 MCS_INACTIVE_ANON,
3611 MCS_ACTIVE_ANON,
3612 MCS_INACTIVE_FILE,
3613 MCS_ACTIVE_FILE,
3614 MCS_UNEVICTABLE,
3615 NR_MCS_STAT,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003616};
3617
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003618struct mcs_total_stat {
3619 s64 stat[NR_MCS_STAT];
3620};
3621
3622struct {
3623 char *local_name;
3624 char *total_name;
3625} memcg_stat_strings[NR_MCS_STAT] = {
3626 {"cache", "total_cache"},
3627 {"rss", "total_rss"},
Balbir Singhd69b0422009-06-17 16:26:34 -07003628 {"mapped_file", "total_mapped_file"},
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003629 {"pgpgin", "total_pgpgin"},
3630 {"pgpgout", "total_pgpgout"},
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003631 {"swap", "total_swap"},
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003632 {"inactive_anon", "total_inactive_anon"},
3633 {"active_anon", "total_active_anon"},
3634 {"inactive_file", "total_inactive_file"},
3635 {"active_file", "total_active_file"},
3636 {"unevictable", "total_unevictable"}
3637};
3638
3639
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003640static void
3641mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003642{
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003643 s64 val;
3644
3645 /* per cpu stat */
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003646 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003647 s->stat[MCS_CACHE] += val * PAGE_SIZE;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003648 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003649 s->stat[MCS_RSS] += val * PAGE_SIZE;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003650 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED);
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -08003651 s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003652 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003653 s->stat[MCS_PGPGIN] += val;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003654 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003655 s->stat[MCS_PGPGOUT] += val;
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003656 if (do_swap_account) {
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003657 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003658 s->stat[MCS_SWAP] += val * PAGE_SIZE;
3659 }
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003660
3661 /* per zone stat */
3662 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
3663 s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
3664 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
3665 s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
3666 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
3667 s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
3668 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
3669 s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
3670 val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
3671 s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003672}
3673
3674static void
3675mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
3676{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003677 struct mem_cgroup *iter;
3678
3679 for_each_mem_cgroup_tree(iter, mem)
3680 mem_cgroup_get_local_stat(iter, s);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003681}
3682
Paul Menagec64745c2008-04-29 01:00:02 -07003683static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
3684 struct cgroup_map_cb *cb)
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003685{
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003686 struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003687 struct mcs_total_stat mystat;
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003688 int i;
3689
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003690 memset(&mystat, 0, sizeof(mystat));
3691 mem_cgroup_get_local_stat(mem_cont, &mystat);
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003692
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003693 for (i = 0; i < NR_MCS_STAT; i++) {
3694 if (i == MCS_SWAP && !do_swap_account)
3695 continue;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003696 cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003697 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08003698
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003699 /* Hierarchical information */
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08003700 {
3701 unsigned long long limit, memsw_limit;
3702 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
3703 cb->fill(cb, "hierarchical_memory_limit", limit);
3704 if (do_swap_account)
3705 cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
3706 }
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003707
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003708 memset(&mystat, 0, sizeof(mystat));
3709 mem_cgroup_get_total_stat(mem_cont, &mystat);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003710 for (i = 0; i < NR_MCS_STAT; i++) {
3711 if (i == MCS_SWAP && !do_swap_account)
3712 continue;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003713 cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003714 }
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003715
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003716#ifdef CONFIG_DEBUG_VM
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08003717 cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003718
3719 {
3720 int nid, zid;
3721 struct mem_cgroup_per_zone *mz;
3722 unsigned long recent_rotated[2] = {0, 0};
3723 unsigned long recent_scanned[2] = {0, 0};
3724
3725 for_each_online_node(nid)
3726 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
3727 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
3728
3729 recent_rotated[0] +=
3730 mz->reclaim_stat.recent_rotated[0];
3731 recent_rotated[1] +=
3732 mz->reclaim_stat.recent_rotated[1];
3733 recent_scanned[0] +=
3734 mz->reclaim_stat.recent_scanned[0];
3735 recent_scanned[1] +=
3736 mz->reclaim_stat.recent_scanned[1];
3737 }
3738 cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
3739 cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
3740 cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
3741 cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
3742 }
3743#endif
3744
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003745 return 0;
3746}
3747
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003748static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
3749{
3750 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3751
3752 return get_swappiness(memcg);
3753}
3754
3755static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
3756 u64 val)
3757{
3758 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3759 struct mem_cgroup *parent;
Li Zefan068b38c2009-01-15 13:51:26 -08003760
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003761 if (val > 100)
3762 return -EINVAL;
3763
3764 if (cgrp->parent == NULL)
3765 return -EINVAL;
3766
3767 parent = mem_cgroup_from_cont(cgrp->parent);
Li Zefan068b38c2009-01-15 13:51:26 -08003768
3769 cgroup_lock();
3770
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003771 /* If under hierarchy, only empty-root can set this value */
3772 if ((parent->use_hierarchy) ||
Li Zefan068b38c2009-01-15 13:51:26 -08003773 (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
3774 cgroup_unlock();
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003775 return -EINVAL;
Li Zefan068b38c2009-01-15 13:51:26 -08003776 }
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003777
3778 spin_lock(&memcg->reclaim_param_lock);
3779 memcg->swappiness = val;
3780 spin_unlock(&memcg->reclaim_param_lock);
3781
Li Zefan068b38c2009-01-15 13:51:26 -08003782 cgroup_unlock();
3783
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003784 return 0;
3785}
3786
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003787static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3788{
3789 struct mem_cgroup_threshold_ary *t;
3790 u64 usage;
3791 int i;
3792
3793 rcu_read_lock();
3794 if (!swap)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003795 t = rcu_dereference(memcg->thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003796 else
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003797 t = rcu_dereference(memcg->memsw_thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003798
3799 if (!t)
3800 goto unlock;
3801
3802 usage = mem_cgroup_usage(memcg, swap);
3803
3804 /*
3805 * current_threshold points to threshold just below usage.
3806 * If it's not true, a threshold was crossed after last
3807 * call of __mem_cgroup_threshold().
3808 */
Phil Carmody5407a562010-05-26 14:42:42 -07003809 i = t->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003810
3811 /*
3812 * Iterate backward over array of thresholds starting from
3813 * current_threshold and check if a threshold is crossed.
3814 * If none of thresholds below usage is crossed, we read
3815 * only one element of the array here.
3816 */
3817 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3818 eventfd_signal(t->entries[i].eventfd, 1);
3819
3820 /* i = current_threshold + 1 */
3821 i++;
3822
3823 /*
3824 * Iterate forward over array of thresholds starting from
3825 * current_threshold+1 and check if a threshold is crossed.
3826 * If none of thresholds above usage is crossed, we read
3827 * only one element of the array here.
3828 */
3829 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3830 eventfd_signal(t->entries[i].eventfd, 1);
3831
3832 /* Update current_threshold */
Phil Carmody5407a562010-05-26 14:42:42 -07003833 t->current_threshold = i - 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003834unlock:
3835 rcu_read_unlock();
3836}
3837
3838static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3839{
Kirill A. Shutemovad4ca5f2010-10-07 12:59:27 -07003840 while (memcg) {
3841 __mem_cgroup_threshold(memcg, false);
3842 if (do_swap_account)
3843 __mem_cgroup_threshold(memcg, true);
3844
3845 memcg = parent_mem_cgroup(memcg);
3846 }
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003847}
3848
3849static int compare_thresholds(const void *a, const void *b)
3850{
3851 const struct mem_cgroup_threshold *_a = a;
3852 const struct mem_cgroup_threshold *_b = b;
3853
3854 return _a->threshold - _b->threshold;
3855}
3856
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003857static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003858{
3859 struct mem_cgroup_eventfd_list *ev;
3860
3861 list_for_each_entry(ev, &mem->oom_notify, list)
3862 eventfd_signal(ev->eventfd, 1);
3863 return 0;
3864}
3865
3866static void mem_cgroup_oom_notify(struct mem_cgroup *mem)
3867{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003868 struct mem_cgroup *iter;
3869
3870 for_each_mem_cgroup_tree(iter, mem)
3871 mem_cgroup_oom_notify_cb(iter);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003872}
3873
3874static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
3875 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003876{
3877 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003878 struct mem_cgroup_thresholds *thresholds;
3879 struct mem_cgroup_threshold_ary *new;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003880 int type = MEMFILE_TYPE(cft->private);
3881 u64 threshold, usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003882 int i, size, ret;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003883
3884 ret = res_counter_memparse_write_strategy(args, &threshold);
3885 if (ret)
3886 return ret;
3887
3888 mutex_lock(&memcg->thresholds_lock);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003889
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003890 if (type == _MEM)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003891 thresholds = &memcg->thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003892 else if (type == _MEMSWAP)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003893 thresholds = &memcg->memsw_thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003894 else
3895 BUG();
3896
3897 usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
3898
3899 /* Check if a threshold crossed before adding a new one */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003900 if (thresholds->primary)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003901 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3902
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003903 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003904
3905 /* Allocate memory for new array of thresholds */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003906 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003907 GFP_KERNEL);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003908 if (!new) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003909 ret = -ENOMEM;
3910 goto unlock;
3911 }
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003912 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003913
3914 /* Copy thresholds (if any) to new array */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003915 if (thresholds->primary) {
3916 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003917 sizeof(struct mem_cgroup_threshold));
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003918 }
3919
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003920 /* Add new threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003921 new->entries[size - 1].eventfd = eventfd;
3922 new->entries[size - 1].threshold = threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003923
3924 /* Sort thresholds. Registering of new threshold isn't time-critical */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003925 sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003926 compare_thresholds, NULL);
3927
3928 /* Find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003929 new->current_threshold = -1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003930 for (i = 0; i < size; i++) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003931 if (new->entries[i].threshold < usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003932 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003933 * new->current_threshold will not be used until
3934 * rcu_assign_pointer(), so it's safe to increment
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003935 * it here.
3936 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003937 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003938 }
3939 }
3940
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003941 /* Free old spare buffer and save old primary buffer as spare */
3942 kfree(thresholds->spare);
3943 thresholds->spare = thresholds->primary;
3944
3945 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003946
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003947 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003948 synchronize_rcu();
3949
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003950unlock:
3951 mutex_unlock(&memcg->thresholds_lock);
3952
3953 return ret;
3954}
3955
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003956static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003957 struct cftype *cft, struct eventfd_ctx *eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003958{
3959 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003960 struct mem_cgroup_thresholds *thresholds;
3961 struct mem_cgroup_threshold_ary *new;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003962 int type = MEMFILE_TYPE(cft->private);
3963 u64 usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003964 int i, j, size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003965
3966 mutex_lock(&memcg->thresholds_lock);
3967 if (type == _MEM)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003968 thresholds = &memcg->thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003969 else if (type == _MEMSWAP)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003970 thresholds = &memcg->memsw_thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003971 else
3972 BUG();
3973
3974 /*
3975 * Something went wrong if we trying to unregister a threshold
3976 * if we don't have thresholds
3977 */
3978 BUG_ON(!thresholds);
3979
3980 usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
3981
3982 /* Check if a threshold crossed before removing */
3983 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3984
3985 /* Calculate new number of threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003986 size = 0;
3987 for (i = 0; i < thresholds->primary->size; i++) {
3988 if (thresholds->primary->entries[i].eventfd != eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003989 size++;
3990 }
3991
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003992 new = thresholds->spare;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003993
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003994 /* Set thresholds array to NULL if we don't have thresholds */
3995 if (!size) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003996 kfree(new);
3997 new = NULL;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003998 goto swap_buffers;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003999 }
4000
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004001 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004002
4003 /* Copy thresholds and find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004004 new->current_threshold = -1;
4005 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
4006 if (thresholds->primary->entries[i].eventfd == eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004007 continue;
4008
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004009 new->entries[j] = thresholds->primary->entries[i];
4010 if (new->entries[j].threshold < usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004011 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004012 * new->current_threshold will not be used
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004013 * until rcu_assign_pointer(), so it's safe to increment
4014 * it here.
4015 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004016 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004017 }
4018 j++;
4019 }
4020
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004021swap_buffers:
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004022 /* Swap primary and spare array */
4023 thresholds->spare = thresholds->primary;
4024 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004025
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004026 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004027 synchronize_rcu();
4028
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004029 mutex_unlock(&memcg->thresholds_lock);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004030}
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004031
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004032static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
4033 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
4034{
4035 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4036 struct mem_cgroup_eventfd_list *event;
4037 int type = MEMFILE_TYPE(cft->private);
4038
4039 BUG_ON(type != _OOM_TYPE);
4040 event = kmalloc(sizeof(*event), GFP_KERNEL);
4041 if (!event)
4042 return -ENOMEM;
4043
4044 mutex_lock(&memcg_oom_mutex);
4045
4046 event->eventfd = eventfd;
4047 list_add(&event->list, &memcg->oom_notify);
4048
4049 /* already in OOM ? */
4050 if (atomic_read(&memcg->oom_lock))
4051 eventfd_signal(eventfd, 1);
4052 mutex_unlock(&memcg_oom_mutex);
4053
4054 return 0;
4055}
4056
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004057static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004058 struct cftype *cft, struct eventfd_ctx *eventfd)
4059{
4060 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4061 struct mem_cgroup_eventfd_list *ev, *tmp;
4062 int type = MEMFILE_TYPE(cft->private);
4063
4064 BUG_ON(type != _OOM_TYPE);
4065
4066 mutex_lock(&memcg_oom_mutex);
4067
4068 list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) {
4069 if (ev->eventfd == eventfd) {
4070 list_del(&ev->list);
4071 kfree(ev);
4072 }
4073 }
4074
4075 mutex_unlock(&memcg_oom_mutex);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004076}
4077
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004078static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
4079 struct cftype *cft, struct cgroup_map_cb *cb)
4080{
4081 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4082
4083 cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable);
4084
4085 if (atomic_read(&mem->oom_lock))
4086 cb->fill(cb, "under_oom", 1);
4087 else
4088 cb->fill(cb, "under_oom", 0);
4089 return 0;
4090}
4091
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004092static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
4093 struct cftype *cft, u64 val)
4094{
4095 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4096 struct mem_cgroup *parent;
4097
4098 /* cannot set to root cgroup and only 0 and 1 are allowed */
4099 if (!cgrp->parent || !((val == 0) || (val == 1)))
4100 return -EINVAL;
4101
4102 parent = mem_cgroup_from_cont(cgrp->parent);
4103
4104 cgroup_lock();
4105 /* oom-kill-disable is a flag for subhierarchy. */
4106 if ((parent->use_hierarchy) ||
4107 (mem->use_hierarchy && !list_empty(&cgrp->children))) {
4108 cgroup_unlock();
4109 return -EINVAL;
4110 }
4111 mem->oom_kill_disable = val;
KAMEZAWA Hiroyuki4d845eb2010-06-29 15:05:18 -07004112 if (!val)
4113 memcg_oom_recover(mem);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004114 cgroup_unlock();
4115 return 0;
4116}
4117
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004118static struct cftype mem_cgroup_files[] = {
4119 {
Balbir Singh0eea1032008-02-07 00:13:57 -08004120 .name = "usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004121 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
Paul Menage2c3daa72008-04-29 00:59:58 -07004122 .read_u64 = mem_cgroup_read,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004123 .register_event = mem_cgroup_usage_register_event,
4124 .unregister_event = mem_cgroup_usage_unregister_event,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004125 },
4126 {
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004127 .name = "max_usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004128 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07004129 .trigger = mem_cgroup_reset,
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004130 .read_u64 = mem_cgroup_read,
4131 },
4132 {
Balbir Singh0eea1032008-02-07 00:13:57 -08004133 .name = "limit_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004134 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
Paul Menage856c13a2008-07-25 01:47:04 -07004135 .write_string = mem_cgroup_write,
Paul Menage2c3daa72008-04-29 00:59:58 -07004136 .read_u64 = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004137 },
4138 {
Balbir Singh296c81d2009-09-23 15:56:36 -07004139 .name = "soft_limit_in_bytes",
4140 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4141 .write_string = mem_cgroup_write,
4142 .read_u64 = mem_cgroup_read,
4143 },
4144 {
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004145 .name = "failcnt",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004146 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07004147 .trigger = mem_cgroup_reset,
Paul Menage2c3daa72008-04-29 00:59:58 -07004148 .read_u64 = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004149 },
Balbir Singh8697d332008-02-07 00:13:59 -08004150 {
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004151 .name = "stat",
Paul Menagec64745c2008-04-29 01:00:02 -07004152 .read_map = mem_control_stat_show,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004153 },
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004154 {
4155 .name = "force_empty",
4156 .trigger = mem_cgroup_force_empty_write,
4157 },
Balbir Singh18f59ea2009-01-07 18:08:07 -08004158 {
4159 .name = "use_hierarchy",
4160 .write_u64 = mem_cgroup_hierarchy_write,
4161 .read_u64 = mem_cgroup_hierarchy_read,
4162 },
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004163 {
4164 .name = "swappiness",
4165 .read_u64 = mem_cgroup_swappiness_read,
4166 .write_u64 = mem_cgroup_swappiness_write,
4167 },
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004168 {
4169 .name = "move_charge_at_immigrate",
4170 .read_u64 = mem_cgroup_move_charge_read,
4171 .write_u64 = mem_cgroup_move_charge_write,
4172 },
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004173 {
4174 .name = "oom_control",
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004175 .read_map = mem_cgroup_oom_control_read,
4176 .write_u64 = mem_cgroup_oom_control_write,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004177 .register_event = mem_cgroup_oom_register_event,
4178 .unregister_event = mem_cgroup_oom_unregister_event,
4179 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4180 },
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004181};
4182
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004183#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4184static struct cftype memsw_cgroup_files[] = {
4185 {
4186 .name = "memsw.usage_in_bytes",
4187 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
4188 .read_u64 = mem_cgroup_read,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004189 .register_event = mem_cgroup_usage_register_event,
4190 .unregister_event = mem_cgroup_usage_unregister_event,
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004191 },
4192 {
4193 .name = "memsw.max_usage_in_bytes",
4194 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
4195 .trigger = mem_cgroup_reset,
4196 .read_u64 = mem_cgroup_read,
4197 },
4198 {
4199 .name = "memsw.limit_in_bytes",
4200 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
4201 .write_string = mem_cgroup_write,
4202 .read_u64 = mem_cgroup_read,
4203 },
4204 {
4205 .name = "memsw.failcnt",
4206 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
4207 .trigger = mem_cgroup_reset,
4208 .read_u64 = mem_cgroup_read,
4209 },
4210};
4211
4212static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
4213{
4214 if (!do_swap_account)
4215 return 0;
4216 return cgroup_add_files(cont, ss, memsw_cgroup_files,
4217 ARRAY_SIZE(memsw_cgroup_files));
4218};
4219#else
4220static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
4221{
4222 return 0;
4223}
4224#endif
4225
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004226static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
4227{
4228 struct mem_cgroup_per_node *pn;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004229 struct mem_cgroup_per_zone *mz;
Christoph Lameterb69408e2008-10-18 20:26:14 -07004230 enum lru_list l;
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07004231 int zone, tmp = node;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004232 /*
4233 * This routine is called against possible nodes.
4234 * But it's BUG to call kmalloc() against offline node.
4235 *
4236 * TODO: this routine can waste much memory for nodes which will
4237 * never be onlined. It's better to use memory hotplug callback
4238 * function.
4239 */
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07004240 if (!node_state(node, N_NORMAL_MEMORY))
4241 tmp = -1;
Jesper Juhl17295c82011-01-13 15:47:42 -08004242 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004243 if (!pn)
4244 return 1;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004245
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004246 mem->info.nodeinfo[node] = pn;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004247 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4248 mz = &pn->zoneinfo[zone];
Christoph Lameterb69408e2008-10-18 20:26:14 -07004249 for_each_lru(l)
4250 INIT_LIST_HEAD(&mz->lists[l]);
Balbir Singhf64c3f52009-09-23 15:56:37 -07004251 mz->usage_in_excess = 0;
Balbir Singh4e416952009-09-23 15:56:39 -07004252 mz->on_tree = false;
4253 mz->mem = mem;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004254 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004255 return 0;
4256}
4257
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004258static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
4259{
4260 kfree(mem->info.nodeinfo[node]);
4261}
4262
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004263static struct mem_cgroup *mem_cgroup_alloc(void)
4264{
4265 struct mem_cgroup *mem;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004266 int size = sizeof(struct mem_cgroup);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004267
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004268 /* Can be very big if MAX_NUMNODES is very big */
Jan Blunckc8dad2b2009-01-07 18:07:53 -08004269 if (size < PAGE_SIZE)
Jesper Juhl17295c82011-01-13 15:47:42 -08004270 mem = kzalloc(size, GFP_KERNEL);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004271 else
Jesper Juhl17295c82011-01-13 15:47:42 -08004272 mem = vzalloc(size);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004273
Dan Carpentere7bbcdf2010-03-23 13:35:12 -07004274 if (!mem)
4275 return NULL;
4276
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004277 mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
Dan Carpenterd2e61b82010-11-11 14:05:12 -08004278 if (!mem->stat)
4279 goto out_free;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07004280 spin_lock_init(&mem->pcp_counter_lock);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004281 return mem;
Dan Carpenterd2e61b82010-11-11 14:05:12 -08004282
4283out_free:
4284 if (size < PAGE_SIZE)
4285 kfree(mem);
4286 else
4287 vfree(mem);
4288 return NULL;
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004289}
4290
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004291/*
4292 * At destroying mem_cgroup, references from swap_cgroup can remain.
4293 * (scanning all at force_empty is too costly...)
4294 *
4295 * Instead of clearing all references at force_empty, we remember
4296 * the number of reference from swap_cgroup and free mem_cgroup when
4297 * it goes down to 0.
4298 *
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004299 * Removal of cgroup itself succeeds regardless of refs from swap.
4300 */
4301
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004302static void __mem_cgroup_free(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004303{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004304 int node;
4305
Balbir Singhf64c3f52009-09-23 15:56:37 -07004306 mem_cgroup_remove_from_trees(mem);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004307 free_css_id(&mem_cgroup_subsys, &mem->css);
4308
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004309 for_each_node_state(node, N_POSSIBLE)
4310 free_mem_cgroup_per_zone_info(mem, node);
4311
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004312 free_percpu(mem->stat);
4313 if (sizeof(struct mem_cgroup) < PAGE_SIZE)
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004314 kfree(mem);
4315 else
4316 vfree(mem);
4317}
4318
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004319static void mem_cgroup_get(struct mem_cgroup *mem)
4320{
4321 atomic_inc(&mem->refcnt);
4322}
4323
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004324static void __mem_cgroup_put(struct mem_cgroup *mem, int count)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004325{
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004326 if (atomic_sub_and_test(count, &mem->refcnt)) {
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004327 struct mem_cgroup *parent = parent_mem_cgroup(mem);
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004328 __mem_cgroup_free(mem);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004329 if (parent)
4330 mem_cgroup_put(parent);
4331 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004332}
4333
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004334static void mem_cgroup_put(struct mem_cgroup *mem)
4335{
4336 __mem_cgroup_put(mem, 1);
4337}
4338
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004339/*
4340 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
4341 */
4342static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
4343{
4344 if (!mem->res.parent)
4345 return NULL;
4346 return mem_cgroup_from_res_counter(mem->res.parent, res);
4347}
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004348
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004349#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4350static void __init enable_swap_cgroup(void)
4351{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08004352 if (!mem_cgroup_disabled() && really_do_swap_account)
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004353 do_swap_account = 1;
4354}
4355#else
4356static void __init enable_swap_cgroup(void)
4357{
4358}
4359#endif
4360
Balbir Singhf64c3f52009-09-23 15:56:37 -07004361static int mem_cgroup_soft_limit_tree_init(void)
4362{
4363 struct mem_cgroup_tree_per_node *rtpn;
4364 struct mem_cgroup_tree_per_zone *rtpz;
4365 int tmp, node, zone;
4366
4367 for_each_node_state(node, N_POSSIBLE) {
4368 tmp = node;
4369 if (!node_state(node, N_NORMAL_MEMORY))
4370 tmp = -1;
4371 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
4372 if (!rtpn)
4373 return 1;
4374
4375 soft_limit_tree.rb_tree_per_node[node] = rtpn;
4376
4377 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4378 rtpz = &rtpn->rb_tree_per_zone[zone];
4379 rtpz->rb_root = RB_ROOT;
4380 spin_lock_init(&rtpz->lock);
4381 }
4382 }
4383 return 0;
4384}
4385
Li Zefan0eb253e2009-01-15 13:51:25 -08004386static struct cgroup_subsys_state * __ref
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004387mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
4388{
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004389 struct mem_cgroup *mem, *parent;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004390 long error = -ENOMEM;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004391 int node;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004392
Jan Blunckc8dad2b2009-01-07 18:07:53 -08004393 mem = mem_cgroup_alloc();
4394 if (!mem)
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004395 return ERR_PTR(error);
Pavel Emelianov78fb7462008-02-07 00:13:51 -08004396
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004397 for_each_node_state(node, N_POSSIBLE)
4398 if (alloc_mem_cgroup_per_zone_info(mem, node))
4399 goto free_out;
Balbir Singhf64c3f52009-09-23 15:56:37 -07004400
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004401 /* root ? */
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004402 if (cont->parent == NULL) {
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08004403 int cpu;
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004404 enable_swap_cgroup();
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004405 parent = NULL;
Balbir Singh4b3bde42009-09-23 15:56:32 -07004406 root_mem_cgroup = mem;
Balbir Singhf64c3f52009-09-23 15:56:37 -07004407 if (mem_cgroup_soft_limit_tree_init())
4408 goto free_out;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08004409 for_each_possible_cpu(cpu) {
4410 struct memcg_stock_pcp *stock =
4411 &per_cpu(memcg_stock, cpu);
4412 INIT_WORK(&stock->work, drain_local_stock);
4413 }
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07004414 hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
Balbir Singh18f59ea2009-01-07 18:08:07 -08004415 } else {
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004416 parent = mem_cgroup_from_cont(cont->parent);
Balbir Singh18f59ea2009-01-07 18:08:07 -08004417 mem->use_hierarchy = parent->use_hierarchy;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004418 mem->oom_kill_disable = parent->oom_kill_disable;
Balbir Singh18f59ea2009-01-07 18:08:07 -08004419 }
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004420
Balbir Singh18f59ea2009-01-07 18:08:07 -08004421 if (parent && parent->use_hierarchy) {
4422 res_counter_init(&mem->res, &parent->res);
4423 res_counter_init(&mem->memsw, &parent->memsw);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004424 /*
4425 * We increment refcnt of the parent to ensure that we can
4426 * safely access it on res_counter_charge/uncharge.
4427 * This refcnt will be decremented when freeing this
4428 * mem_cgroup(see mem_cgroup_put).
4429 */
4430 mem_cgroup_get(parent);
Balbir Singh18f59ea2009-01-07 18:08:07 -08004431 } else {
4432 res_counter_init(&mem->res, NULL);
4433 res_counter_init(&mem->memsw, NULL);
4434 }
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004435 mem->last_scanned_child = 0;
KOSAKI Motohiro2733c062009-01-07 18:08:23 -08004436 spin_lock_init(&mem->reclaim_param_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004437 INIT_LIST_HEAD(&mem->oom_notify);
Balbir Singh6d61ef42009-01-07 18:08:06 -08004438
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004439 if (parent)
4440 mem->swappiness = get_swappiness(parent);
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004441 atomic_set(&mem->refcnt, 1);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004442 mem->move_charge_at_immigrate = 0;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004443 mutex_init(&mem->thresholds_lock);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004444 return &mem->css;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004445free_out:
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004446 __mem_cgroup_free(mem);
Balbir Singh4b3bde42009-09-23 15:56:32 -07004447 root_mem_cgroup = NULL;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004448 return ERR_PTR(error);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004449}
4450
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07004451static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08004452 struct cgroup *cont)
4453{
4454 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07004455
4456 return mem_cgroup_force_empty(mem, false);
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08004457}
4458
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004459static void mem_cgroup_destroy(struct cgroup_subsys *ss,
4460 struct cgroup *cont)
4461{
Daisuke Nishimurac268e992009-01-15 13:51:13 -08004462 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
Daisuke Nishimurac268e992009-01-15 13:51:13 -08004463
Daisuke Nishimurac268e992009-01-15 13:51:13 -08004464 mem_cgroup_put(mem);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004465}
4466
4467static int mem_cgroup_populate(struct cgroup_subsys *ss,
4468 struct cgroup *cont)
4469{
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004470 int ret;
4471
4472 ret = cgroup_add_files(cont, ss, mem_cgroup_files,
4473 ARRAY_SIZE(mem_cgroup_files));
4474
4475 if (!ret)
4476 ret = register_memsw_files(cont, ss);
4477 return ret;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004478}
4479
Daisuke Nishimura02491442010-03-10 15:22:17 -08004480#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004481/* Handlers for move charge at task migration. */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004482#define PRECHARGE_COUNT_AT_ONCE 256
4483static int mem_cgroup_do_precharge(unsigned long count)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004484{
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004485 int ret = 0;
4486 int batch_count = PRECHARGE_COUNT_AT_ONCE;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004487 struct mem_cgroup *mem = mc.to;
4488
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004489 if (mem_cgroup_is_root(mem)) {
4490 mc.precharge += count;
4491 /* we don't need css_get for root */
4492 return ret;
4493 }
4494 /* try to charge at once */
4495 if (count > 1) {
4496 struct res_counter *dummy;
4497 /*
4498 * "mem" cannot be under rmdir() because we've already checked
4499 * by cgroup_lock_live_cgroup() that it is not removed and we
4500 * are still under the same cgroup_mutex. So we can postpone
4501 * css_get().
4502 */
4503 if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
4504 goto one_by_one;
4505 if (do_swap_account && res_counter_charge(&mem->memsw,
4506 PAGE_SIZE * count, &dummy)) {
4507 res_counter_uncharge(&mem->res, PAGE_SIZE * count);
4508 goto one_by_one;
4509 }
4510 mc.precharge += count;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004511 return ret;
4512 }
4513one_by_one:
4514 /* fall back to one by one charge */
4515 while (count--) {
4516 if (signal_pending(current)) {
4517 ret = -EINTR;
4518 break;
4519 }
4520 if (!batch_count--) {
4521 batch_count = PRECHARGE_COUNT_AT_ONCE;
4522 cond_resched();
4523 }
Andrea Arcangeliec168512011-01-13 15:46:56 -08004524 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false,
4525 PAGE_SIZE);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004526 if (ret || !mem)
4527 /* mem_cgroup_clear_mc() will do uncharge later */
4528 return -ENOMEM;
4529 mc.precharge++;
4530 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004531 return ret;
4532}
4533
4534/**
4535 * is_target_pte_for_mc - check a pte whether it is valid for move charge
4536 * @vma: the vma the pte to be checked belongs
4537 * @addr: the address corresponding to the pte to be checked
4538 * @ptent: the pte to be checked
Daisuke Nishimura02491442010-03-10 15:22:17 -08004539 * @target: the pointer the target page or swap ent will be stored(can be NULL)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004540 *
4541 * Returns
4542 * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
4543 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4544 * move charge. if @target is not NULL, the page is stored in target->page
4545 * with extra refcnt got(Callers should handle it).
Daisuke Nishimura02491442010-03-10 15:22:17 -08004546 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4547 * target for charge migration. if @target is not NULL, the entry is stored
4548 * in target->ent.
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004549 *
4550 * Called with pte lock held.
4551 */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004552union mc_target {
4553 struct page *page;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004554 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004555};
4556
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004557enum mc_target_type {
4558 MC_TARGET_NONE, /* not used */
4559 MC_TARGET_PAGE,
Daisuke Nishimura02491442010-03-10 15:22:17 -08004560 MC_TARGET_SWAP,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004561};
4562
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004563static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4564 unsigned long addr, pte_t ptent)
4565{
4566 struct page *page = vm_normal_page(vma, addr, ptent);
4567
4568 if (!page || !page_mapped(page))
4569 return NULL;
4570 if (PageAnon(page)) {
4571 /* we don't move shared anon */
4572 if (!move_anon() || page_mapcount(page) > 2)
4573 return NULL;
Daisuke Nishimura87946a72010-05-26 14:42:39 -07004574 } else if (!move_file())
4575 /* we ignore mapcount for file pages */
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004576 return NULL;
4577 if (!get_page_unless_zero(page))
4578 return NULL;
4579
4580 return page;
4581}
4582
4583static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4584 unsigned long addr, pte_t ptent, swp_entry_t *entry)
4585{
4586 int usage_count;
4587 struct page *page = NULL;
4588 swp_entry_t ent = pte_to_swp_entry(ptent);
4589
4590 if (!move_anon() || non_swap_entry(ent))
4591 return NULL;
4592 usage_count = mem_cgroup_count_swap_user(ent, &page);
4593 if (usage_count > 1) { /* we don't move shared anon */
4594 if (page)
4595 put_page(page);
4596 return NULL;
4597 }
4598 if (do_swap_account)
4599 entry->val = ent.val;
4600
4601 return page;
4602}
4603
Daisuke Nishimura87946a72010-05-26 14:42:39 -07004604static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4605 unsigned long addr, pte_t ptent, swp_entry_t *entry)
4606{
4607 struct page *page = NULL;
4608 struct inode *inode;
4609 struct address_space *mapping;
4610 pgoff_t pgoff;
4611
4612 if (!vma->vm_file) /* anonymous vma */
4613 return NULL;
4614 if (!move_file())
4615 return NULL;
4616
4617 inode = vma->vm_file->f_path.dentry->d_inode;
4618 mapping = vma->vm_file->f_mapping;
4619 if (pte_none(ptent))
4620 pgoff = linear_page_index(vma, addr);
4621 else /* pte_file(ptent) is true */
4622 pgoff = pte_to_pgoff(ptent);
4623
4624 /* page is moved even if it's not RSS of this task(page-faulted). */
4625 if (!mapping_cap_swap_backed(mapping)) { /* normal file */
4626 page = find_get_page(mapping, pgoff);
4627 } else { /* shmem/tmpfs file. we should take account of swap too. */
4628 swp_entry_t ent;
4629 mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent);
4630 if (do_swap_account)
4631 entry->val = ent.val;
4632 }
4633
4634 return page;
4635}
4636
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004637static int is_target_pte_for_mc(struct vm_area_struct *vma,
4638 unsigned long addr, pte_t ptent, union mc_target *target)
4639{
Daisuke Nishimura02491442010-03-10 15:22:17 -08004640 struct page *page = NULL;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004641 struct page_cgroup *pc;
4642 int ret = 0;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004643 swp_entry_t ent = { .val = 0 };
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004644
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004645 if (pte_present(ptent))
4646 page = mc_handle_present_pte(vma, addr, ptent);
4647 else if (is_swap_pte(ptent))
4648 page = mc_handle_swap_pte(vma, addr, ptent, &ent);
Daisuke Nishimura87946a72010-05-26 14:42:39 -07004649 else if (pte_none(ptent) || pte_file(ptent))
4650 page = mc_handle_file_pte(vma, addr, ptent, &ent);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004651
4652 if (!page && !ent.val)
Daisuke Nishimura02491442010-03-10 15:22:17 -08004653 return 0;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004654 if (page) {
4655 pc = lookup_page_cgroup(page);
4656 /*
4657 * Do only loose check w/o page_cgroup lock.
4658 * mem_cgroup_move_account() checks the pc is valid or not under
4659 * the lock.
4660 */
4661 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
4662 ret = MC_TARGET_PAGE;
4663 if (target)
4664 target->page = page;
4665 }
4666 if (!ret || !target)
4667 put_page(page);
4668 }
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004669 /* There is a swap entry and a page doesn't exist or isn't charged */
4670 if (ent.val && !ret &&
KAMEZAWA Hiroyuki7f0f1542010-05-11 14:06:58 -07004671 css_id(&mc.from->css) == lookup_swap_cgroup(ent)) {
4672 ret = MC_TARGET_SWAP;
4673 if (target)
4674 target->ent = ent;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004675 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004676 return ret;
4677}
4678
4679static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4680 unsigned long addr, unsigned long end,
4681 struct mm_walk *walk)
4682{
4683 struct vm_area_struct *vma = walk->private;
4684 pte_t *pte;
4685 spinlock_t *ptl;
4686
Andrea Arcangeliec168512011-01-13 15:46:56 -08004687 VM_BUG_ON(pmd_trans_huge(*pmd));
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004688 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4689 for (; addr != end; pte++, addr += PAGE_SIZE)
4690 if (is_target_pte_for_mc(vma, addr, *pte, NULL))
4691 mc.precharge++; /* increment precharge temporarily */
4692 pte_unmap_unlock(pte - 1, ptl);
4693 cond_resched();
4694
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004695 return 0;
4696}
4697
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004698static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4699{
4700 unsigned long precharge;
4701 struct vm_area_struct *vma;
4702
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004703 down_read(&mm->mmap_sem);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004704 for (vma = mm->mmap; vma; vma = vma->vm_next) {
4705 struct mm_walk mem_cgroup_count_precharge_walk = {
4706 .pmd_entry = mem_cgroup_count_precharge_pte_range,
4707 .mm = mm,
4708 .private = vma,
4709 };
4710 if (is_vm_hugetlb_page(vma))
4711 continue;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004712 walk_page_range(vma->vm_start, vma->vm_end,
4713 &mem_cgroup_count_precharge_walk);
4714 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004715 up_read(&mm->mmap_sem);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004716
4717 precharge = mc.precharge;
4718 mc.precharge = 0;
4719
4720 return precharge;
4721}
4722
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004723static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4724{
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004725 unsigned long precharge = mem_cgroup_count_precharge(mm);
4726
4727 VM_BUG_ON(mc.moving_task);
4728 mc.moving_task = current;
4729 return mem_cgroup_do_precharge(precharge);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004730}
4731
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004732/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
4733static void __mem_cgroup_clear_mc(void)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004734{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004735 struct mem_cgroup *from = mc.from;
4736 struct mem_cgroup *to = mc.to;
4737
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004738 /* we must uncharge all the leftover precharges from mc.to */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004739 if (mc.precharge) {
4740 __mem_cgroup_cancel_charge(mc.to, mc.precharge);
4741 mc.precharge = 0;
4742 }
4743 /*
4744 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4745 * we must uncharge here.
4746 */
4747 if (mc.moved_charge) {
4748 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
4749 mc.moved_charge = 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004750 }
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004751 /* we must fixup refcnts and charges */
4752 if (mc.moved_swap) {
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004753 /* uncharge swap account from the old cgroup */
4754 if (!mem_cgroup_is_root(mc.from))
4755 res_counter_uncharge(&mc.from->memsw,
4756 PAGE_SIZE * mc.moved_swap);
4757 __mem_cgroup_put(mc.from, mc.moved_swap);
4758
4759 if (!mem_cgroup_is_root(mc.to)) {
4760 /*
4761 * we charged both to->res and to->memsw, so we should
4762 * uncharge to->res.
4763 */
4764 res_counter_uncharge(&mc.to->res,
4765 PAGE_SIZE * mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004766 }
4767 /* we've already done mem_cgroup_get(mc.to) */
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004768 mc.moved_swap = 0;
4769 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004770 memcg_oom_recover(from);
4771 memcg_oom_recover(to);
4772 wake_up_all(&mc.waitq);
4773}
4774
4775static void mem_cgroup_clear_mc(void)
4776{
4777 struct mem_cgroup *from = mc.from;
4778
4779 /*
4780 * we must clear moving_task before waking up waiters at the end of
4781 * task migration.
4782 */
4783 mc.moving_task = NULL;
4784 __mem_cgroup_clear_mc();
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004785 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004786 mc.from = NULL;
4787 mc.to = NULL;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004788 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07004789 mem_cgroup_end_move(from);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004790}
4791
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004792static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
4793 struct cgroup *cgroup,
4794 struct task_struct *p,
4795 bool threadgroup)
4796{
4797 int ret = 0;
4798 struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup);
4799
4800 if (mem->move_charge_at_immigrate) {
4801 struct mm_struct *mm;
4802 struct mem_cgroup *from = mem_cgroup_from_task(p);
4803
4804 VM_BUG_ON(from == mem);
4805
4806 mm = get_task_mm(p);
4807 if (!mm)
4808 return 0;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004809 /* We move charges only when we move a owner of the mm */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004810 if (mm->owner == p) {
4811 VM_BUG_ON(mc.from);
4812 VM_BUG_ON(mc.to);
4813 VM_BUG_ON(mc.precharge);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004814 VM_BUG_ON(mc.moved_charge);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004815 VM_BUG_ON(mc.moved_swap);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07004816 mem_cgroup_start_move(from);
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004817 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004818 mc.from = from;
4819 mc.to = mem;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004820 spin_unlock(&mc.lock);
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004821 /* We set mc.moving_task later */
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004822
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004823 ret = mem_cgroup_precharge_mc(mm);
4824 if (ret)
4825 mem_cgroup_clear_mc();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004826 }
4827 mmput(mm);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004828 }
4829 return ret;
4830}
4831
4832static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
4833 struct cgroup *cgroup,
4834 struct task_struct *p,
4835 bool threadgroup)
4836{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004837 mem_cgroup_clear_mc();
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004838}
4839
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004840static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4841 unsigned long addr, unsigned long end,
4842 struct mm_walk *walk)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004843{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004844 int ret = 0;
4845 struct vm_area_struct *vma = walk->private;
4846 pte_t *pte;
4847 spinlock_t *ptl;
4848
4849retry:
Andrea Arcangeliec168512011-01-13 15:46:56 -08004850 VM_BUG_ON(pmd_trans_huge(*pmd));
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004851 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4852 for (; addr != end; addr += PAGE_SIZE) {
4853 pte_t ptent = *(pte++);
4854 union mc_target target;
4855 int type;
4856 struct page *page;
4857 struct page_cgroup *pc;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004858 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004859
4860 if (!mc.precharge)
4861 break;
4862
4863 type = is_target_pte_for_mc(vma, addr, ptent, &target);
4864 switch (type) {
4865 case MC_TARGET_PAGE:
4866 page = target.page;
4867 if (isolate_lru_page(page))
4868 goto put;
4869 pc = lookup_page_cgroup(page);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004870 if (!mem_cgroup_move_account(pc,
4871 mc.from, mc.to, false)) {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004872 mc.precharge--;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004873 /* we uncharge from mc.from later. */
4874 mc.moved_charge++;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004875 }
4876 putback_lru_page(page);
4877put: /* is_target_pte_for_mc() gets the page */
4878 put_page(page);
4879 break;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004880 case MC_TARGET_SWAP:
4881 ent = target.ent;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004882 if (!mem_cgroup_move_swap_account(ent,
4883 mc.from, mc.to, false)) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08004884 mc.precharge--;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004885 /* we fixup refcnts and charges later. */
4886 mc.moved_swap++;
4887 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08004888 break;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004889 default:
4890 break;
4891 }
4892 }
4893 pte_unmap_unlock(pte - 1, ptl);
4894 cond_resched();
4895
4896 if (addr != end) {
4897 /*
4898 * We have consumed all precharges we got in can_attach().
4899 * We try charge one by one, but don't do any additional
4900 * charges to mc.to if we have failed in charge once in attach()
4901 * phase.
4902 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004903 ret = mem_cgroup_do_precharge(1);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004904 if (!ret)
4905 goto retry;
4906 }
4907
4908 return ret;
4909}
4910
4911static void mem_cgroup_move_charge(struct mm_struct *mm)
4912{
4913 struct vm_area_struct *vma;
4914
4915 lru_add_drain_all();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004916retry:
4917 if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
4918 /*
4919 * Someone who are holding the mmap_sem might be waiting in
4920 * waitq. So we cancel all extra charges, wake up all waiters,
4921 * and retry. Because we cancel precharges, we might not be able
4922 * to move enough charges, but moving charge is a best-effort
4923 * feature anyway, so it wouldn't be a big problem.
4924 */
4925 __mem_cgroup_clear_mc();
4926 cond_resched();
4927 goto retry;
4928 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004929 for (vma = mm->mmap; vma; vma = vma->vm_next) {
4930 int ret;
4931 struct mm_walk mem_cgroup_move_charge_walk = {
4932 .pmd_entry = mem_cgroup_move_charge_pte_range,
4933 .mm = mm,
4934 .private = vma,
4935 };
4936 if (is_vm_hugetlb_page(vma))
4937 continue;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004938 ret = walk_page_range(vma->vm_start, vma->vm_end,
4939 &mem_cgroup_move_charge_walk);
4940 if (ret)
4941 /*
4942 * means we have consumed all precharges and failed in
4943 * doing additional charge. Just abandon here.
4944 */
4945 break;
4946 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004947 up_read(&mm->mmap_sem);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004948}
4949
Balbir Singh67e465a2008-02-07 00:13:54 -08004950static void mem_cgroup_move_task(struct cgroup_subsys *ss,
4951 struct cgroup *cont,
4952 struct cgroup *old_cont,
Ben Blumbe367d02009-09-23 15:56:31 -07004953 struct task_struct *p,
4954 bool threadgroup)
Balbir Singh67e465a2008-02-07 00:13:54 -08004955{
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004956 struct mm_struct *mm;
4957
4958 if (!mc.to)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004959 /* no need to move charge */
4960 return;
4961
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004962 mm = get_task_mm(p);
4963 if (mm) {
4964 mem_cgroup_move_charge(mm);
4965 mmput(mm);
4966 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004967 mem_cgroup_clear_mc();
Balbir Singh67e465a2008-02-07 00:13:54 -08004968}
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07004969#else /* !CONFIG_MMU */
4970static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
4971 struct cgroup *cgroup,
4972 struct task_struct *p,
4973 bool threadgroup)
4974{
4975 return 0;
4976}
4977static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
4978 struct cgroup *cgroup,
4979 struct task_struct *p,
4980 bool threadgroup)
4981{
4982}
4983static void mem_cgroup_move_task(struct cgroup_subsys *ss,
4984 struct cgroup *cont,
4985 struct cgroup *old_cont,
4986 struct task_struct *p,
4987 bool threadgroup)
4988{
4989}
4990#endif
Balbir Singh67e465a2008-02-07 00:13:54 -08004991
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004992struct cgroup_subsys mem_cgroup_subsys = {
4993 .name = "memory",
4994 .subsys_id = mem_cgroup_subsys_id,
4995 .create = mem_cgroup_create,
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08004996 .pre_destroy = mem_cgroup_pre_destroy,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004997 .destroy = mem_cgroup_destroy,
4998 .populate = mem_cgroup_populate,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004999 .can_attach = mem_cgroup_can_attach,
5000 .cancel_attach = mem_cgroup_cancel_attach,
Balbir Singh67e465a2008-02-07 00:13:54 -08005001 .attach = mem_cgroup_move_task,
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005002 .early_init = 0,
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07005003 .use_id = 1,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005004};
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08005005
5006#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
Michal Hockoa42c3902010-11-24 12:57:08 -08005007static int __init enable_swap_account(char *s)
5008{
5009 /* consider enabled if no parameter or 1 is given */
5010 if (!s || !strcmp(s, "1"))
5011 really_do_swap_account = 1;
5012 else if (!strcmp(s, "0"))
5013 really_do_swap_account = 0;
5014 return 1;
5015}
5016__setup("swapaccount", enable_swap_account);
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08005017
5018static int __init disable_swap_account(char *s)
5019{
Michal Hockoa42c3902010-11-24 12:57:08 -08005020 enable_swap_account("0");
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08005021 return 1;
5022}
5023__setup("noswapaccount", disable_swap_account);
5024#endif