blob: 8ab8410314363871da25fa9f613578f5ebe01783 [file] [log] [blame]
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001/* memcontrol.c - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08006 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08009 * Memory thresholds
10 * Copyright (C) 2009 Nokia Corporation
11 * Author: Kirill A. Shutemov
12 *
Balbir Singh8cdea7c2008-02-07 00:13:50 -080013 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 */
23
24#include <linux/res_counter.h>
25#include <linux/memcontrol.h>
26#include <linux/cgroup.h>
Pavel Emelianov78fb7462008-02-07 00:13:51 -080027#include <linux/mm.h>
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -080028#include <linux/hugetlb.h>
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -080029#include <linux/pagemap.h>
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080030#include <linux/smp.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080031#include <linux/page-flags.h>
Balbir Singh66e17072008-02-07 00:13:56 -080032#include <linux/backing-dev.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080033#include <linux/bit_spinlock.h>
34#include <linux/rcupdate.h>
Balbir Singhe2224322009-04-02 16:57:39 -070035#include <linux/limits.h>
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -080036#include <linux/mutex.h>
Balbir Singhf64c3f52009-09-23 15:56:37 -070037#include <linux/rbtree.h>
Balbir Singhb6ac57d2008-04-29 01:00:19 -070038#include <linux/slab.h>
Balbir Singh66e17072008-02-07 00:13:56 -080039#include <linux/swap.h>
Daisuke Nishimura02491442010-03-10 15:22:17 -080040#include <linux/swapops.h>
Balbir Singh66e17072008-02-07 00:13:56 -080041#include <linux/spinlock.h>
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080042#include <linux/eventfd.h>
43#include <linux/sort.h>
Balbir Singh66e17072008-02-07 00:13:56 -080044#include <linux/fs.h>
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -080045#include <linux/seq_file.h>
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -070046#include <linux/vmalloc.h>
Christoph Lameterb69408e2008-10-18 20:26:14 -070047#include <linux/mm_inline.h>
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -070048#include <linux/page_cgroup.h>
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -080049#include <linux/cpu.h>
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -070050#include <linux/oom.h>
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -080051#include "internal.h"
Balbir Singh8cdea7c2008-02-07 00:13:50 -080052
Balbir Singh8697d332008-02-07 00:13:59 -080053#include <asm/uaccess.h>
54
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -070055#include <trace/events/vmscan.h>
56
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070057struct cgroup_subsys mem_cgroup_subsys __read_mostly;
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070058#define MEM_CGROUP_RECLAIM_RETRIES 5
Balbir Singh4b3bde42009-09-23 15:56:32 -070059struct mem_cgroup *root_mem_cgroup __read_mostly;
Balbir Singh8cdea7c2008-02-07 00:13:50 -080060
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080061#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
Li Zefan338c8432009-06-17 16:27:15 -070062/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080063int do_swap_account __read_mostly;
Michal Hockoa42c3902010-11-24 12:57:08 -080064
65/* for remember boot option*/
66#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED
67static int really_do_swap_account __initdata = 1;
68#else
69static int really_do_swap_account __initdata = 0;
70#endif
71
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080072#else
73#define do_swap_account (0)
74#endif
75
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -080076/*
77 * Per memcg event counter is incremented at every pagein/pageout. This counter
78 * is used for trigger some periodic events. This is straightforward and better
79 * than using jiffies etc. to handle periodic memcg event.
80 *
81 * These values will be used as !((event) & ((1 <<(thresh)) - 1))
82 */
83#define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */
84#define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080085
Balbir Singh8cdea7c2008-02-07 00:13:50 -080086/*
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080087 * Statistics for memory cgroup.
88 */
89enum mem_cgroup_stat_index {
90 /*
91 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
92 */
93 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
Balbir Singhd69b0422009-06-17 16:26:34 -070094 MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -080095 MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
Balaji Rao55e462b2008-05-01 04:35:12 -070096 MEM_CGROUP_STAT_PGPGIN_COUNT, /* # of pages paged in */
97 MEM_CGROUP_STAT_PGPGOUT_COUNT, /* # of pages paged out */
Balbir Singh0c3e73e2009-09-23 15:56:42 -070098 MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -070099 MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */
100 /* incremented at every pagein/pageout */
101 MEM_CGROUP_EVENTS = MEM_CGROUP_STAT_DATA,
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -0700102 MEM_CGROUP_ON_MOVE, /* someone is moving account between groups */
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800103
104 MEM_CGROUP_STAT_NSTATS,
105};
106
107struct mem_cgroup_stat_cpu {
108 s64 count[MEM_CGROUP_STAT_NSTATS];
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800109};
110
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800111/*
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800112 * per-zone information in memory controller.
113 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800114struct mem_cgroup_per_zone {
KAMEZAWA Hiroyuki072c56c12008-02-07 00:14:39 -0800115 /*
116 * spin_lock to protect the per cgroup LRU
117 */
Christoph Lameterb69408e2008-10-18 20:26:14 -0700118 struct list_head lists[NR_LRU_LISTS];
119 unsigned long count[NR_LRU_LISTS];
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -0800120
121 struct zone_reclaim_stat reclaim_stat;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700122 struct rb_node tree_node; /* RB tree node */
123 unsigned long long usage_in_excess;/* Set to the value by which */
124 /* the soft limit is exceeded*/
125 bool on_tree;
Balbir Singh4e416952009-09-23 15:56:39 -0700126 struct mem_cgroup *mem; /* Back pointer, we cannot */
127 /* use container_of */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800128};
129/* Macro for accessing counter */
130#define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)])
131
132struct mem_cgroup_per_node {
133 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
134};
135
136struct mem_cgroup_lru_info {
137 struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
138};
139
140/*
Balbir Singhf64c3f52009-09-23 15:56:37 -0700141 * Cgroups above their limits are maintained in a RB-Tree, independent of
142 * their hierarchy representation
143 */
144
145struct mem_cgroup_tree_per_zone {
146 struct rb_root rb_root;
147 spinlock_t lock;
148};
149
150struct mem_cgroup_tree_per_node {
151 struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
152};
153
154struct mem_cgroup_tree {
155 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
156};
157
158static struct mem_cgroup_tree soft_limit_tree __read_mostly;
159
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800160struct mem_cgroup_threshold {
161 struct eventfd_ctx *eventfd;
162 u64 threshold;
163};
164
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700165/* For threshold */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800166struct mem_cgroup_threshold_ary {
167 /* An array index points to threshold just below usage. */
Phil Carmody5407a562010-05-26 14:42:42 -0700168 int current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800169 /* Size of entries[] */
170 unsigned int size;
171 /* Array of thresholds */
172 struct mem_cgroup_threshold entries[0];
173};
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700174
175struct mem_cgroup_thresholds {
176 /* Primary thresholds array */
177 struct mem_cgroup_threshold_ary *primary;
178 /*
179 * Spare threshold array.
180 * This is needed to make mem_cgroup_unregister_event() "never fail".
181 * It must be able to store at least primary->size - 1 entries.
182 */
183 struct mem_cgroup_threshold_ary *spare;
184};
185
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700186/* for OOM */
187struct mem_cgroup_eventfd_list {
188 struct list_head list;
189 struct eventfd_ctx *eventfd;
190};
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800191
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800192static void mem_cgroup_threshold(struct mem_cgroup *mem);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700193static void mem_cgroup_oom_notify(struct mem_cgroup *mem);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800194
Balbir Singhf64c3f52009-09-23 15:56:37 -0700195/*
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800196 * The memory controller data structure. The memory controller controls both
197 * page cache and RSS per cgroup. We would eventually like to provide
198 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
199 * to help the administrator determine what knobs to tune.
200 *
201 * TODO: Add a water mark for the memory controller. Reclaim will begin when
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800202 * we hit the water mark. May be even add a low water mark, such that
203 * no reclaim occurs from a cgroup at it's low water mark, this is
204 * a feature that will be implemented much later in the future.
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800205 */
206struct mem_cgroup {
207 struct cgroup_subsys_state css;
208 /*
209 * the counter to account for memory usage
210 */
211 struct res_counter res;
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800212 /*
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800213 * the counter to account for mem+swap usage.
214 */
215 struct res_counter memsw;
216 /*
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800217 * Per cgroup active and inactive list, similar to the
218 * per zone LRU lists.
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800219 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800220 struct mem_cgroup_lru_info info;
KAMEZAWA Hiroyuki072c56c12008-02-07 00:14:39 -0800221
KOSAKI Motohiro2733c062009-01-07 18:08:23 -0800222 /*
223 protect against reclaim related member.
224 */
225 spinlock_t reclaim_param_lock;
226
Balbir Singh6d61ef42009-01-07 18:08:06 -0800227 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200228 * While reclaiming in a hierarchy, we cache the last child we
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -0700229 * reclaimed from.
Balbir Singh6d61ef42009-01-07 18:08:06 -0800230 */
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -0700231 int last_scanned_child;
Balbir Singh18f59ea2009-01-07 18:08:07 -0800232 /*
233 * Should the accounting and control be hierarchical, per subtree?
234 */
235 bool use_hierarchy;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -0800236 atomic_t oom_lock;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800237 atomic_t refcnt;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800238
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -0800239 unsigned int swappiness;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -0700240 /* OOM-Killer disable */
241 int oom_kill_disable;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -0800242
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -0700243 /* set when res.limit == memsw.limit */
244 bool memsw_is_minimum;
245
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800246 /* protect arrays of thresholds */
247 struct mutex thresholds_lock;
248
249 /* thresholds for memory usage. RCU-protected */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700250 struct mem_cgroup_thresholds thresholds;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -0700251
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800252 /* thresholds for mem+swap usage. RCU-protected */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700253 struct mem_cgroup_thresholds memsw_thresholds;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -0700254
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700255 /* For oom notifier event fd */
256 struct list_head oom_notify;
257
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800258 /*
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800259 * Should we move charges of a task when a task is moved into this
260 * mem_cgroup ? And what type of charges should we move ?
261 */
262 unsigned long move_charge_at_immigrate;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800263 /*
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800264 * percpu counter.
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800265 */
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800266 struct mem_cgroup_stat_cpu *stat;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700267 /*
268 * used when a cpu is offlined or other synchronizations
269 * See mem_cgroup_read_stat().
270 */
271 struct mem_cgroup_stat_cpu nocpu_base;
272 spinlock_t pcp_counter_lock;
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800273};
274
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800275/* Stuffs for move charges at task migration. */
276/*
277 * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
278 * left-shifted bitmap of these types.
279 */
280enum move_type {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800281 MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700282 MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800283 NR_MOVE_TYPE,
284};
285
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800286/* "mc" and its members are protected by cgroup_mutex */
287static struct move_charge_struct {
Daisuke Nishimurab1dd6932010-11-24 12:57:06 -0800288 spinlock_t lock; /* for from, to */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800289 struct mem_cgroup *from;
290 struct mem_cgroup *to;
291 unsigned long precharge;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -0800292 unsigned long moved_charge;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -0800293 unsigned long moved_swap;
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800294 struct task_struct *moving_task; /* a task moving charges */
295 wait_queue_head_t waitq; /* a waitq for other context */
296} mc = {
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -0700297 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800298 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
299};
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800300
Daisuke Nishimura90254a62010-05-26 14:42:38 -0700301static bool move_anon(void)
302{
303 return test_bit(MOVE_CHARGE_TYPE_ANON,
304 &mc.to->move_charge_at_immigrate);
305}
306
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700307static bool move_file(void)
308{
309 return test_bit(MOVE_CHARGE_TYPE_FILE,
310 &mc.to->move_charge_at_immigrate);
311}
312
Balbir Singh4e416952009-09-23 15:56:39 -0700313/*
314 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
315 * limit reclaim to prevent infinite loops, if they ever occur.
316 */
317#define MEM_CGROUP_MAX_RECLAIM_LOOPS (100)
318#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2)
319
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800320enum charge_type {
321 MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
322 MEM_CGROUP_CHARGE_TYPE_MAPPED,
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700323 MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700324 MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -0800325 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -0700326 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700327 NR_CHARGE_TYPE,
328};
329
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700330/* only for here (for easy reading.) */
331#define PCGF_CACHE (1UL << PCG_CACHE)
332#define PCGF_USED (1UL << PCG_USED)
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700333#define PCGF_LOCK (1UL << PCG_LOCK)
Balbir Singh4b3bde42009-09-23 15:56:32 -0700334/* Not used, but added here for completeness */
335#define PCGF_ACCT (1UL << PCG_ACCT)
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800336
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800337/* for encoding cft->private value on file */
338#define _MEM (0)
339#define _MEMSWAP (1)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700340#define _OOM_TYPE (2)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800341#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
342#define MEMFILE_TYPE(val) (((val) >> 16) & 0xffff)
343#define MEMFILE_ATTR(val) ((val) & 0xffff)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700344/* Used for OOM nofiier */
345#define OOM_CONTROL (0)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800346
Balbir Singh75822b42009-09-23 15:56:38 -0700347/*
348 * Reclaim flags for mem_cgroup_hierarchical_reclaim
349 */
350#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0
351#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
352#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1
353#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
Balbir Singh4e416952009-09-23 15:56:39 -0700354#define MEM_CGROUP_RECLAIM_SOFT_BIT 0x2
355#define MEM_CGROUP_RECLAIM_SOFT (1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
Balbir Singh75822b42009-09-23 15:56:38 -0700356
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800357static void mem_cgroup_get(struct mem_cgroup *mem);
358static void mem_cgroup_put(struct mem_cgroup *mem);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -0800359static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -0800360static void drain_all_stock_async(void);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800361
Balbir Singhf64c3f52009-09-23 15:56:37 -0700362static struct mem_cgroup_per_zone *
363mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
364{
365 return &mem->info.nodeinfo[nid]->zoneinfo[zid];
366}
367
Wu Fengguangd3242362009-12-16 12:19:59 +0100368struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
369{
370 return &mem->css;
371}
372
Balbir Singhf64c3f52009-09-23 15:56:37 -0700373static struct mem_cgroup_per_zone *
374page_cgroup_zoneinfo(struct page_cgroup *pc)
375{
376 struct mem_cgroup *mem = pc->mem_cgroup;
377 int nid = page_cgroup_nid(pc);
378 int zid = page_cgroup_zid(pc);
379
380 if (!mem)
381 return NULL;
382
383 return mem_cgroup_zoneinfo(mem, nid, zid);
384}
385
386static struct mem_cgroup_tree_per_zone *
387soft_limit_tree_node_zone(int nid, int zid)
388{
389 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
390}
391
392static struct mem_cgroup_tree_per_zone *
393soft_limit_tree_from_page(struct page *page)
394{
395 int nid = page_to_nid(page);
396 int zid = page_zonenum(page);
397
398 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
399}
400
401static void
Balbir Singh4e416952009-09-23 15:56:39 -0700402__mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
Balbir Singhf64c3f52009-09-23 15:56:37 -0700403 struct mem_cgroup_per_zone *mz,
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700404 struct mem_cgroup_tree_per_zone *mctz,
405 unsigned long long new_usage_in_excess)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700406{
407 struct rb_node **p = &mctz->rb_root.rb_node;
408 struct rb_node *parent = NULL;
409 struct mem_cgroup_per_zone *mz_node;
410
411 if (mz->on_tree)
412 return;
413
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700414 mz->usage_in_excess = new_usage_in_excess;
415 if (!mz->usage_in_excess)
416 return;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700417 while (*p) {
418 parent = *p;
419 mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
420 tree_node);
421 if (mz->usage_in_excess < mz_node->usage_in_excess)
422 p = &(*p)->rb_left;
423 /*
424 * We can't avoid mem cgroups that are over their soft
425 * limit by the same amount
426 */
427 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
428 p = &(*p)->rb_right;
429 }
430 rb_link_node(&mz->tree_node, parent, p);
431 rb_insert_color(&mz->tree_node, &mctz->rb_root);
432 mz->on_tree = true;
Balbir Singh4e416952009-09-23 15:56:39 -0700433}
434
435static void
436__mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
437 struct mem_cgroup_per_zone *mz,
438 struct mem_cgroup_tree_per_zone *mctz)
439{
440 if (!mz->on_tree)
441 return;
442 rb_erase(&mz->tree_node, &mctz->rb_root);
443 mz->on_tree = false;
444}
445
446static void
Balbir Singhf64c3f52009-09-23 15:56:37 -0700447mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
448 struct mem_cgroup_per_zone *mz,
449 struct mem_cgroup_tree_per_zone *mctz)
450{
451 spin_lock(&mctz->lock);
Balbir Singh4e416952009-09-23 15:56:39 -0700452 __mem_cgroup_remove_exceeded(mem, mz, mctz);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700453 spin_unlock(&mctz->lock);
454}
455
Balbir Singhf64c3f52009-09-23 15:56:37 -0700456
457static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
458{
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700459 unsigned long long excess;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700460 struct mem_cgroup_per_zone *mz;
461 struct mem_cgroup_tree_per_zone *mctz;
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700462 int nid = page_to_nid(page);
463 int zid = page_zonenum(page);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700464 mctz = soft_limit_tree_from_page(page);
465
466 /*
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700467 * Necessary to update all ancestors when hierarchy is used.
468 * because their event counter is not touched.
Balbir Singhf64c3f52009-09-23 15:56:37 -0700469 */
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700470 for (; mem; mem = parent_mem_cgroup(mem)) {
471 mz = mem_cgroup_zoneinfo(mem, nid, zid);
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700472 excess = res_counter_soft_limit_excess(&mem->res);
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700473 /*
474 * We have to update the tree if mz is on RB-tree or
475 * mem is over its softlimit.
476 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700477 if (excess || mz->on_tree) {
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700478 spin_lock(&mctz->lock);
479 /* if on-tree, remove it */
480 if (mz->on_tree)
481 __mem_cgroup_remove_exceeded(mem, mz, mctz);
482 /*
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700483 * Insert again. mz->usage_in_excess will be updated.
484 * If excess is 0, no tree ops.
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700485 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700486 __mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700487 spin_unlock(&mctz->lock);
488 }
Balbir Singhf64c3f52009-09-23 15:56:37 -0700489 }
490}
491
492static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
493{
494 int node, zone;
495 struct mem_cgroup_per_zone *mz;
496 struct mem_cgroup_tree_per_zone *mctz;
497
498 for_each_node_state(node, N_POSSIBLE) {
499 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
500 mz = mem_cgroup_zoneinfo(mem, node, zone);
501 mctz = soft_limit_tree_node_zone(node, zone);
502 mem_cgroup_remove_exceeded(mem, mz, mctz);
503 }
504 }
505}
506
Balbir Singh4e416952009-09-23 15:56:39 -0700507static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem)
508{
509 return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT;
510}
511
512static struct mem_cgroup_per_zone *
513__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
514{
515 struct rb_node *rightmost = NULL;
KAMEZAWA Hiroyuki26251ea2009-10-01 15:44:08 -0700516 struct mem_cgroup_per_zone *mz;
Balbir Singh4e416952009-09-23 15:56:39 -0700517
518retry:
KAMEZAWA Hiroyuki26251ea2009-10-01 15:44:08 -0700519 mz = NULL;
Balbir Singh4e416952009-09-23 15:56:39 -0700520 rightmost = rb_last(&mctz->rb_root);
521 if (!rightmost)
522 goto done; /* Nothing to reclaim from */
523
524 mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
525 /*
526 * Remove the node now but someone else can add it back,
527 * we will to add it back at the end of reclaim to its correct
528 * position in the tree.
529 */
530 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
531 if (!res_counter_soft_limit_excess(&mz->mem->res) ||
532 !css_tryget(&mz->mem->css))
533 goto retry;
534done:
535 return mz;
536}
537
538static struct mem_cgroup_per_zone *
539mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
540{
541 struct mem_cgroup_per_zone *mz;
542
543 spin_lock(&mctz->lock);
544 mz = __mem_cgroup_largest_soft_limit_node(mctz);
545 spin_unlock(&mctz->lock);
546 return mz;
547}
548
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700549/*
550 * Implementation Note: reading percpu statistics for memcg.
551 *
552 * Both of vmstat[] and percpu_counter has threshold and do periodic
553 * synchronization to implement "quick" read. There are trade-off between
554 * reading cost and precision of value. Then, we may have a chance to implement
555 * a periodic synchronizion of counter in memcg's counter.
556 *
557 * But this _read() function is used for user interface now. The user accounts
558 * memory usage by memory cgroup and he _always_ requires exact value because
559 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
560 * have to visit all online cpus and make sum. So, for now, unnecessary
561 * synchronization is not implemented. (just implemented for cpu hotplug)
562 *
563 * If there are kernel internal actions which can make use of some not-exact
564 * value, and reading all cpu value can be performance bottleneck in some
565 * common workload, threashold and synchonization as vmstat[] should be
566 * implemented.
567 */
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800568static s64 mem_cgroup_read_stat(struct mem_cgroup *mem,
569 enum mem_cgroup_stat_index idx)
570{
571 int cpu;
572 s64 val = 0;
573
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700574 get_online_cpus();
575 for_each_online_cpu(cpu)
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800576 val += per_cpu(mem->stat->count[idx], cpu);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700577#ifdef CONFIG_HOTPLUG_CPU
578 spin_lock(&mem->pcp_counter_lock);
579 val += mem->nocpu_base.count[idx];
580 spin_unlock(&mem->pcp_counter_lock);
581#endif
582 put_online_cpus();
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800583 return val;
584}
585
586static s64 mem_cgroup_local_usage(struct mem_cgroup *mem)
587{
588 s64 ret;
589
590 ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
591 ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
592 return ret;
593}
594
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700595static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
596 bool charge)
597{
598 int val = (charge) ? 1 : -1;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800599 this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700600}
601
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700602static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
603 struct page_cgroup *pc,
604 bool charge)
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800605{
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700606 int val = (charge) ? 1 : -1;
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800607
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800608 preempt_disable();
609
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700610 if (PageCgroupCache(pc))
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800611 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], val);
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800612 else
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800613 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], val);
Balaji Rao55e462b2008-05-01 04:35:12 -0700614
615 if (charge)
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800616 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]);
Balaji Rao55e462b2008-05-01 04:35:12 -0700617 else
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800618 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]);
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800619 __this_cpu_inc(mem->stat->count[MEM_CGROUP_EVENTS]);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800620
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800621 preempt_enable();
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800622}
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800623
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700624static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
Christoph Lameterb69408e2008-10-18 20:26:14 -0700625 enum lru_list idx)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800626{
627 int nid, zid;
628 struct mem_cgroup_per_zone *mz;
629 u64 total = 0;
630
631 for_each_online_node(nid)
632 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
633 mz = mem_cgroup_zoneinfo(mem, nid, zid);
634 total += MEM_CGROUP_ZSTAT(mz, idx);
635 }
636 return total;
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800637}
638
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800639static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift)
640{
641 s64 val;
642
643 val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]);
644
645 return !(val & ((1 << event_mask_shift) - 1));
646}
647
648/*
649 * Check events in order.
650 *
651 */
652static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
653{
654 /* threshold event is triggered in finer grain than soft limit */
655 if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) {
656 mem_cgroup_threshold(mem);
657 if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH)))
658 mem_cgroup_update_tree(mem, page);
659 }
660}
661
Hugh Dickinsd5b69e32008-03-04 14:29:10 -0800662static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800663{
664 return container_of(cgroup_subsys_state(cont,
665 mem_cgroup_subsys_id), struct mem_cgroup,
666 css);
667}
668
Balbir Singhcf475ad2008-04-29 01:00:16 -0700669struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800670{
Balbir Singh31a78f22008-09-28 23:09:31 +0100671 /*
672 * mm_update_next_owner() may clear mm->owner to NULL
673 * if it races with swapoff, page migration, etc.
674 * So this can be called with p == NULL.
675 */
676 if (unlikely(!p))
677 return NULL;
678
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800679 return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
680 struct mem_cgroup, css);
681}
682
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800683static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
684{
685 struct mem_cgroup *mem = NULL;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700686
687 if (!mm)
688 return NULL;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800689 /*
690 * Because we have no locks, mm->owner's may be being moved to other
691 * cgroup. We use css_tryget() here even if this looks
692 * pessimistic (rather than adding locks here).
693 */
694 rcu_read_lock();
695 do {
696 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
697 if (unlikely(!mem))
698 break;
699 } while (!css_tryget(&mem->css));
700 rcu_read_unlock();
701 return mem;
702}
703
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700704/* The caller has to guarantee "mem" exists before calling this */
705static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700706{
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700707 struct cgroup_subsys_state *css;
708 int found;
709
710 if (!mem) /* ROOT cgroup has the smallest ID */
711 return root_mem_cgroup; /*css_put/get against root is ignored*/
712 if (!mem->use_hierarchy) {
713 if (css_tryget(&mem->css))
714 return mem;
715 return NULL;
716 }
717 rcu_read_lock();
718 /*
719 * searching a memory cgroup which has the smallest ID under given
720 * ROOT cgroup. (ID >= 1)
721 */
722 css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found);
723 if (css && css_tryget(css))
724 mem = container_of(css, struct mem_cgroup, css);
725 else
726 mem = NULL;
727 rcu_read_unlock();
728 return mem;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700729}
730
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700731static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter,
732 struct mem_cgroup *root,
733 bool cond)
734{
735 int nextid = css_id(&iter->css) + 1;
736 int found;
737 int hierarchy_used;
738 struct cgroup_subsys_state *css;
739
740 hierarchy_used = iter->use_hierarchy;
741
742 css_put(&iter->css);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700743 /* If no ROOT, walk all, ignore hierarchy */
744 if (!cond || (root && !hierarchy_used))
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700745 return NULL;
746
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700747 if (!root)
748 root = root_mem_cgroup;
749
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700750 do {
751 iter = NULL;
752 rcu_read_lock();
753
754 css = css_get_next(&mem_cgroup_subsys, nextid,
755 &root->css, &found);
756 if (css && css_tryget(css))
757 iter = container_of(css, struct mem_cgroup, css);
758 rcu_read_unlock();
759 /* If css is NULL, no more cgroups will be found */
760 nextid = found + 1;
761 } while (css && !iter);
762
763 return iter;
764}
765/*
766 * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please
767 * be careful that "break" loop is not allowed. We have reference count.
768 * Instead of that modify "cond" to be false and "continue" to exit the loop.
769 */
770#define for_each_mem_cgroup_tree_cond(iter, root, cond) \
771 for (iter = mem_cgroup_start_loop(root);\
772 iter != NULL;\
773 iter = mem_cgroup_get_next(iter, root, cond))
774
775#define for_each_mem_cgroup_tree(iter, root) \
776 for_each_mem_cgroup_tree_cond(iter, root, true)
777
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700778#define for_each_mem_cgroup_all(iter) \
779 for_each_mem_cgroup_tree_cond(iter, NULL, true)
780
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700781
Balbir Singh4b3bde42009-09-23 15:56:32 -0700782static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
783{
784 return (mem == root_mem_cgroup);
785}
786
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800787/*
788 * Following LRU functions are allowed to be used without PCG_LOCK.
789 * Operations are called by routine of global LRU independently from memcg.
790 * What we have to take care of here is validness of pc->mem_cgroup.
791 *
792 * Changes to pc->mem_cgroup happens when
793 * 1. charge
794 * 2. moving account
795 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
796 * It is added to LRU before charge.
797 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
798 * When moving account, the page is not on LRU. It's isolated.
799 */
800
801void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800802{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800803 struct page_cgroup *pc;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800804 struct mem_cgroup_per_zone *mz;
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700805
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800806 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800807 return;
808 pc = lookup_page_cgroup(page);
809 /* can happen while we handle swapcache. */
Balbir Singh4b3bde42009-09-23 15:56:32 -0700810 if (!TestClearPageCgroupAcctLRU(pc))
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800811 return;
Balbir Singh4b3bde42009-09-23 15:56:32 -0700812 VM_BUG_ON(!pc->mem_cgroup);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800813 /*
814 * We don't check PCG_USED bit. It's cleared when the "page" is finally
815 * removed from global LRU.
816 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800817 mz = page_cgroup_zoneinfo(pc);
Christoph Lameterb69408e2008-10-18 20:26:14 -0700818 MEM_CGROUP_ZSTAT(mz, lru) -= 1;
Balbir Singh4b3bde42009-09-23 15:56:32 -0700819 if (mem_cgroup_is_root(pc->mem_cgroup))
820 return;
821 VM_BUG_ON(list_empty(&pc->lru));
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800822 list_del_init(&pc->lru);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800823}
824
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800825void mem_cgroup_del_lru(struct page *page)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800826{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800827 mem_cgroup_del_lru_list(page, page_lru(page));
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800828}
829
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800830void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
Balbir Singh66e17072008-02-07 00:13:56 -0800831{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800832 struct mem_cgroup_per_zone *mz;
833 struct page_cgroup *pc;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800834
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800835 if (mem_cgroup_disabled())
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700836 return;
Christoph Lameterb69408e2008-10-18 20:26:14 -0700837
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800838 pc = lookup_page_cgroup(page);
Daisuke Nishimurabd112db2009-01-15 13:51:11 -0800839 /*
840 * Used bit is set without atomic ops but after smp_wmb().
841 * For making pc->mem_cgroup visible, insert smp_rmb() here.
842 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800843 smp_rmb();
Balbir Singh4b3bde42009-09-23 15:56:32 -0700844 /* unused or root page is not rotated. */
845 if (!PageCgroupUsed(pc) || mem_cgroup_is_root(pc->mem_cgroup))
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800846 return;
847 mz = page_cgroup_zoneinfo(pc);
Christoph Lameterb69408e2008-10-18 20:26:14 -0700848 list_move(&pc->lru, &mz->lists[lru]);
Balbir Singh66e17072008-02-07 00:13:56 -0800849}
850
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800851void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
852{
853 struct page_cgroup *pc;
854 struct mem_cgroup_per_zone *mz;
855
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800856 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800857 return;
858 pc = lookup_page_cgroup(page);
Balbir Singh4b3bde42009-09-23 15:56:32 -0700859 VM_BUG_ON(PageCgroupAcctLRU(pc));
Daisuke Nishimurabd112db2009-01-15 13:51:11 -0800860 /*
861 * Used bit is set without atomic ops but after smp_wmb().
862 * For making pc->mem_cgroup visible, insert smp_rmb() here.
863 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800864 smp_rmb();
865 if (!PageCgroupUsed(pc))
866 return;
867
868 mz = page_cgroup_zoneinfo(pc);
869 MEM_CGROUP_ZSTAT(mz, lru) += 1;
Balbir Singh4b3bde42009-09-23 15:56:32 -0700870 SetPageCgroupAcctLRU(pc);
871 if (mem_cgroup_is_root(pc->mem_cgroup))
872 return;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800873 list_add(&pc->lru, &mz->lists[lru]);
874}
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800875
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800876/*
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800877 * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to
878 * lru because the page may.be reused after it's fully uncharged (because of
879 * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge
880 * it again. This function is only used to charge SwapCache. It's done under
881 * lock_page and expected that zone->lru_lock is never held.
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800882 */
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800883static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page)
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800884{
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800885 unsigned long flags;
886 struct zone *zone = page_zone(page);
887 struct page_cgroup *pc = lookup_page_cgroup(page);
888
889 spin_lock_irqsave(&zone->lru_lock, flags);
890 /*
891 * Forget old LRU when this page_cgroup is *not* used. This Used bit
892 * is guarded by lock_page() because the page is SwapCache.
893 */
894 if (!PageCgroupUsed(pc))
895 mem_cgroup_del_lru_list(page, page_lru(page));
896 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800897}
898
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800899static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
900{
901 unsigned long flags;
902 struct zone *zone = page_zone(page);
903 struct page_cgroup *pc = lookup_page_cgroup(page);
904
905 spin_lock_irqsave(&zone->lru_lock, flags);
906 /* link when the page is linked to LRU but page_cgroup isn't */
Balbir Singh4b3bde42009-09-23 15:56:32 -0700907 if (PageLRU(page) && !PageCgroupAcctLRU(pc))
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800908 mem_cgroup_add_lru_list(page, page_lru(page));
909 spin_unlock_irqrestore(&zone->lru_lock, flags);
910}
911
912
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800913void mem_cgroup_move_lists(struct page *page,
914 enum lru_list from, enum lru_list to)
915{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -0800916 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800917 return;
918 mem_cgroup_del_lru_list(page, from);
919 mem_cgroup_add_lru_list(page, to);
920}
921
David Rientjes4c4a2212008-02-07 00:14:06 -0800922int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
923{
924 int ret;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700925 struct mem_cgroup *curr = NULL;
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -0700926 struct task_struct *p;
David Rientjes4c4a2212008-02-07 00:14:06 -0800927
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -0700928 p = find_lock_task_mm(task);
929 if (!p)
930 return 0;
931 curr = try_get_mem_cgroup_from_mm(p->mm);
932 task_unlock(p);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700933 if (!curr)
934 return 0;
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -0800935 /*
936 * We should check use_hierarchy of "mem" not "curr". Because checking
937 * use_hierarchy of "curr" here make this function true if hierarchy is
938 * enabled in "curr" and "curr" is a child of "mem" in *cgroup*
939 * hierarchy(even if use_hierarchy is disabled in "mem").
940 */
941 if (mem->use_hierarchy)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700942 ret = css_is_ancestor(&curr->css, &mem->css);
943 else
944 ret = (curr == mem);
945 css_put(&curr->css);
David Rientjes4c4a2212008-02-07 00:14:06 -0800946 return ret;
947}
948
KOSAKI Motohiroc772be92009-01-07 18:08:25 -0800949static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800950{
951 unsigned long active;
952 unsigned long inactive;
KOSAKI Motohiroc772be92009-01-07 18:08:25 -0800953 unsigned long gb;
954 unsigned long inactive_ratio;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800955
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700956 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
957 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800958
KOSAKI Motohiroc772be92009-01-07 18:08:25 -0800959 gb = (inactive + active) >> (30 - PAGE_SHIFT);
960 if (gb)
961 inactive_ratio = int_sqrt(10 * gb);
962 else
963 inactive_ratio = 1;
964
965 if (present_pages) {
966 present_pages[0] = inactive;
967 present_pages[1] = active;
968 }
969
970 return inactive_ratio;
971}
972
973int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
974{
975 unsigned long active;
976 unsigned long inactive;
977 unsigned long present_pages[2];
978 unsigned long inactive_ratio;
979
980 inactive_ratio = calc_inactive_ratio(memcg, present_pages);
981
982 inactive = present_pages[0];
983 active = present_pages[1];
984
985 if (inactive * inactive_ratio < active)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800986 return 1;
987
988 return 0;
989}
990
Rik van Riel56e49d22009-06-16 15:32:28 -0700991int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
992{
993 unsigned long active;
994 unsigned long inactive;
995
996 inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
997 active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
998
999 return (active > inactive);
1000}
1001
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -08001002unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
1003 struct zone *zone,
1004 enum lru_list lru)
1005{
KOSAKI Motohiro13d7e3a2010-08-10 18:03:06 -07001006 int nid = zone_to_nid(zone);
KOSAKI Motohiroa3d8e052009-01-07 18:08:19 -08001007 int zid = zone_idx(zone);
1008 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
1009
1010 return MEM_CGROUP_ZSTAT(mz, lru);
1011}
1012
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -08001013struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
1014 struct zone *zone)
1015{
KOSAKI Motohiro13d7e3a2010-08-10 18:03:06 -07001016 int nid = zone_to_nid(zone);
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -08001017 int zid = zone_idx(zone);
1018 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
1019
1020 return &mz->reclaim_stat;
1021}
1022
1023struct zone_reclaim_stat *
1024mem_cgroup_get_reclaim_stat_from_page(struct page *page)
1025{
1026 struct page_cgroup *pc;
1027 struct mem_cgroup_per_zone *mz;
1028
1029 if (mem_cgroup_disabled())
1030 return NULL;
1031
1032 pc = lookup_page_cgroup(page);
Daisuke Nishimurabd112db2009-01-15 13:51:11 -08001033 /*
1034 * Used bit is set without atomic ops but after smp_wmb().
1035 * For making pc->mem_cgroup visible, insert smp_rmb() here.
1036 */
1037 smp_rmb();
1038 if (!PageCgroupUsed(pc))
1039 return NULL;
1040
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -08001041 mz = page_cgroup_zoneinfo(pc);
1042 if (!mz)
1043 return NULL;
1044
1045 return &mz->reclaim_stat;
1046}
1047
Balbir Singh66e17072008-02-07 00:13:56 -08001048unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
1049 struct list_head *dst,
1050 unsigned long *scanned, int order,
1051 int mode, struct zone *z,
1052 struct mem_cgroup *mem_cont,
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001053 int active, int file)
Balbir Singh66e17072008-02-07 00:13:56 -08001054{
1055 unsigned long nr_taken = 0;
1056 struct page *page;
1057 unsigned long scan;
1058 LIST_HEAD(pc_list);
1059 struct list_head *src;
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001060 struct page_cgroup *pc, *tmp;
KOSAKI Motohiro13d7e3a2010-08-10 18:03:06 -07001061 int nid = zone_to_nid(z);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001062 int zid = zone_idx(z);
1063 struct mem_cgroup_per_zone *mz;
Johannes Weinerb7c46d12009-09-21 17:02:56 -07001064 int lru = LRU_FILE * file + active;
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001065 int ret;
Balbir Singh66e17072008-02-07 00:13:56 -08001066
Balbir Singhcf475ad2008-04-29 01:00:16 -07001067 BUG_ON(!mem_cont);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001068 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
Christoph Lameterb69408e2008-10-18 20:26:14 -07001069 src = &mz->lists[lru];
Balbir Singh66e17072008-02-07 00:13:56 -08001070
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001071 scan = 0;
1072 list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
Hugh Dickins436c65412008-02-07 00:14:12 -08001073 if (scan >= nr_to_scan)
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001074 break;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001075
1076 page = pc->page;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07001077 if (unlikely(!PageCgroupUsed(pc)))
1078 continue;
Hugh Dickins436c65412008-02-07 00:14:12 -08001079 if (unlikely(!PageLRU(page)))
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001080 continue;
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001081
Hugh Dickins436c65412008-02-07 00:14:12 -08001082 scan++;
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001083 ret = __isolate_lru_page(page, mode, file);
1084 switch (ret) {
1085 case 0:
Balbir Singh66e17072008-02-07 00:13:56 -08001086 list_move(&page->lru, dst);
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001087 mem_cgroup_del_lru(page);
Rik van Riel2c888cf2011-01-13 15:47:13 -08001088 nr_taken += hpage_nr_pages(page);
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001089 break;
1090 case -EBUSY:
1091 /* we don't affect global LRU but rotate in our LRU */
1092 mem_cgroup_rotate_lru_list(page, page_lru(page));
1093 break;
1094 default:
1095 break;
Balbir Singh66e17072008-02-07 00:13:56 -08001096 }
1097 }
1098
Balbir Singh66e17072008-02-07 00:13:56 -08001099 *scanned = scan;
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -07001100
1101 trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken,
1102 0, 0, 0, mode);
1103
Balbir Singh66e17072008-02-07 00:13:56 -08001104 return nr_taken;
1105}
1106
Balbir Singh6d61ef42009-01-07 18:08:06 -08001107#define mem_cgroup_from_res_counter(counter, member) \
1108 container_of(counter, struct mem_cgroup, member)
1109
Daisuke Nishimurab85a96c2009-01-07 18:08:12 -08001110static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
1111{
1112 if (do_swap_account) {
1113 if (res_counter_check_under_limit(&mem->res) &&
1114 res_counter_check_under_limit(&mem->memsw))
1115 return true;
1116 } else
1117 if (res_counter_check_under_limit(&mem->res))
1118 return true;
1119 return false;
1120}
1121
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001122static unsigned int get_swappiness(struct mem_cgroup *memcg)
1123{
1124 struct cgroup *cgrp = memcg->css.cgroup;
1125 unsigned int swappiness;
1126
1127 /* root ? */
1128 if (cgrp->parent == NULL)
1129 return vm_swappiness;
1130
1131 spin_lock(&memcg->reclaim_param_lock);
1132 swappiness = memcg->swappiness;
1133 spin_unlock(&memcg->reclaim_param_lock);
1134
1135 return swappiness;
1136}
1137
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001138static void mem_cgroup_start_move(struct mem_cgroup *mem)
1139{
1140 int cpu;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001141
1142 get_online_cpus();
1143 spin_lock(&mem->pcp_counter_lock);
1144 for_each_online_cpu(cpu)
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001145 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001146 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
1147 spin_unlock(&mem->pcp_counter_lock);
1148 put_online_cpus();
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001149
1150 synchronize_rcu();
1151}
1152
1153static void mem_cgroup_end_move(struct mem_cgroup *mem)
1154{
1155 int cpu;
1156
1157 if (!mem)
1158 return;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001159 get_online_cpus();
1160 spin_lock(&mem->pcp_counter_lock);
1161 for_each_online_cpu(cpu)
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001162 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001163 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
1164 spin_unlock(&mem->pcp_counter_lock);
1165 put_online_cpus();
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001166}
1167/*
1168 * 2 routines for checking "mem" is under move_account() or not.
1169 *
1170 * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used
1171 * for avoiding race in accounting. If true,
1172 * pc->mem_cgroup may be overwritten.
1173 *
1174 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
1175 * under hierarchy of moving cgroups. This is for
1176 * waiting at hith-memory prressure caused by "move".
1177 */
1178
1179static bool mem_cgroup_stealed(struct mem_cgroup *mem)
1180{
1181 VM_BUG_ON(!rcu_read_lock_held());
1182 return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
1183}
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001184
1185static bool mem_cgroup_under_move(struct mem_cgroup *mem)
1186{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001187 struct mem_cgroup *from;
1188 struct mem_cgroup *to;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001189 bool ret = false;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001190 /*
1191 * Unlike task_move routines, we access mc.to, mc.from not under
1192 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1193 */
1194 spin_lock(&mc.lock);
1195 from = mc.from;
1196 to = mc.to;
1197 if (!from)
1198 goto unlock;
1199 if (from == mem || to == mem
1200 || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css))
1201 || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css)))
1202 ret = true;
1203unlock:
1204 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001205 return ret;
1206}
1207
1208static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem)
1209{
1210 if (mc.moving_task && current != mc.moving_task) {
1211 if (mem_cgroup_under_move(mem)) {
1212 DEFINE_WAIT(wait);
1213 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1214 /* moving charge context might have finished. */
1215 if (mc.moving_task)
1216 schedule();
1217 finish_wait(&mc.waitq, &wait);
1218 return true;
1219 }
1220 }
1221 return false;
1222}
1223
Balbir Singhe2224322009-04-02 16:57:39 -07001224/**
Kirill A. Shutemov6a6135b2010-03-10 15:22:25 -08001225 * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
Balbir Singhe2224322009-04-02 16:57:39 -07001226 * @memcg: The memory cgroup that went over limit
1227 * @p: Task that is going to be killed
1228 *
1229 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1230 * enabled
1231 */
1232void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1233{
1234 struct cgroup *task_cgrp;
1235 struct cgroup *mem_cgrp;
1236 /*
1237 * Need a buffer in BSS, can't rely on allocations. The code relies
1238 * on the assumption that OOM is serialized for memory controller.
1239 * If this assumption is broken, revisit this code.
1240 */
1241 static char memcg_name[PATH_MAX];
1242 int ret;
1243
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -08001244 if (!memcg || !p)
Balbir Singhe2224322009-04-02 16:57:39 -07001245 return;
1246
1247
1248 rcu_read_lock();
1249
1250 mem_cgrp = memcg->css.cgroup;
1251 task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1252
1253 ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1254 if (ret < 0) {
1255 /*
1256 * Unfortunately, we are unable to convert to a useful name
1257 * But we'll still print out the usage information
1258 */
1259 rcu_read_unlock();
1260 goto done;
1261 }
1262 rcu_read_unlock();
1263
1264 printk(KERN_INFO "Task in %s killed", memcg_name);
1265
1266 rcu_read_lock();
1267 ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1268 if (ret < 0) {
1269 rcu_read_unlock();
1270 goto done;
1271 }
1272 rcu_read_unlock();
1273
1274 /*
1275 * Continues from above, so we don't need an KERN_ level
1276 */
1277 printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
1278done:
1279
1280 printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
1281 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1282 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1283 res_counter_read_u64(&memcg->res, RES_FAILCNT));
1284 printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
1285 "failcnt %llu\n",
1286 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1287 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1288 res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1289}
1290
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001291/*
1292 * This function returns the number of memcg under hierarchy tree. Returns
1293 * 1(self count) if no children.
1294 */
1295static int mem_cgroup_count_children(struct mem_cgroup *mem)
1296{
1297 int num = 0;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001298 struct mem_cgroup *iter;
1299
1300 for_each_mem_cgroup_tree(iter, mem)
1301 num++;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001302 return num;
1303}
1304
Balbir Singh6d61ef42009-01-07 18:08:06 -08001305/*
David Rientjesa63d83f2010-08-09 17:19:46 -07001306 * Return the memory (and swap, if configured) limit for a memcg.
1307 */
1308u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
1309{
1310 u64 limit;
1311 u64 memsw;
1312
Johannes Weinerf3e8eb72011-01-13 15:47:39 -08001313 limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
1314 limit += total_swap_pages << PAGE_SHIFT;
1315
David Rientjesa63d83f2010-08-09 17:19:46 -07001316 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1317 /*
1318 * If memsw is finite and limits the amount of swap space available
1319 * to this memcg, return that limit.
1320 */
1321 return min(limit, memsw);
1322}
1323
1324/*
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001325 * Visit the first child (need not be the first child as per the ordering
1326 * of the cgroup list, since we track last_scanned_child) of @mem and use
1327 * that to reclaim free pages from.
1328 */
1329static struct mem_cgroup *
1330mem_cgroup_select_victim(struct mem_cgroup *root_mem)
1331{
1332 struct mem_cgroup *ret = NULL;
1333 struct cgroup_subsys_state *css;
1334 int nextid, found;
1335
1336 if (!root_mem->use_hierarchy) {
1337 css_get(&root_mem->css);
1338 ret = root_mem;
1339 }
1340
1341 while (!ret) {
1342 rcu_read_lock();
1343 nextid = root_mem->last_scanned_child + 1;
1344 css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
1345 &found);
1346 if (css && css_tryget(css))
1347 ret = container_of(css, struct mem_cgroup, css);
1348
1349 rcu_read_unlock();
1350 /* Updates scanning parameter */
1351 spin_lock(&root_mem->reclaim_param_lock);
1352 if (!css) {
1353 /* this means start scan from ID:1 */
1354 root_mem->last_scanned_child = 0;
1355 } else
1356 root_mem->last_scanned_child = found;
1357 spin_unlock(&root_mem->reclaim_param_lock);
1358 }
1359
1360 return ret;
1361}
1362
1363/*
1364 * Scan the hierarchy if needed to reclaim memory. We remember the last child
1365 * we reclaimed from, so that we don't end up penalizing one child extensively
1366 * based on its position in the children list.
Balbir Singh6d61ef42009-01-07 18:08:06 -08001367 *
1368 * root_mem is the original ancestor that we've been reclaim from.
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001369 *
1370 * We give up and return to the caller when we visit root_mem twice.
1371 * (other groups can be removed while we're walking....)
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001372 *
1373 * If shrink==true, for avoiding to free too much, this returns immedieately.
Balbir Singh6d61ef42009-01-07 18:08:06 -08001374 */
1375static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
Balbir Singh4e416952009-09-23 15:56:39 -07001376 struct zone *zone,
Balbir Singh75822b42009-09-23 15:56:38 -07001377 gfp_t gfp_mask,
1378 unsigned long reclaim_options)
Balbir Singh6d61ef42009-01-07 18:08:06 -08001379{
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001380 struct mem_cgroup *victim;
1381 int ret, total = 0;
1382 int loop = 0;
Balbir Singh75822b42009-09-23 15:56:38 -07001383 bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
1384 bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
Balbir Singh4e416952009-09-23 15:56:39 -07001385 bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
1386 unsigned long excess = mem_cgroup_get_excess(root_mem);
Balbir Singh6d61ef42009-01-07 18:08:06 -08001387
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07001388 /* If memsw_is_minimum==1, swap-out is of-no-use. */
1389 if (root_mem->memsw_is_minimum)
1390 noswap = true;
1391
Balbir Singh4e416952009-09-23 15:56:39 -07001392 while (1) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001393 victim = mem_cgroup_select_victim(root_mem);
Balbir Singh4e416952009-09-23 15:56:39 -07001394 if (victim == root_mem) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001395 loop++;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001396 if (loop >= 1)
1397 drain_all_stock_async();
Balbir Singh4e416952009-09-23 15:56:39 -07001398 if (loop >= 2) {
1399 /*
1400 * If we have not been able to reclaim
1401 * anything, it might because there are
1402 * no reclaimable pages under this hierarchy
1403 */
1404 if (!check_soft || !total) {
1405 css_put(&victim->css);
1406 break;
1407 }
1408 /*
1409 * We want to do more targetted reclaim.
1410 * excess >> 2 is not to excessive so as to
1411 * reclaim too much, nor too less that we keep
1412 * coming back to reclaim from this cgroup
1413 */
1414 if (total >= (excess >> 2) ||
1415 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
1416 css_put(&victim->css);
1417 break;
1418 }
1419 }
1420 }
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08001421 if (!mem_cgroup_local_usage(victim)) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001422 /* this cgroup's local usage == 0 */
1423 css_put(&victim->css);
Balbir Singh6d61ef42009-01-07 18:08:06 -08001424 continue;
1425 }
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001426 /* we use swappiness of local cgroup */
Balbir Singh4e416952009-09-23 15:56:39 -07001427 if (check_soft)
1428 ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
KOSAKI Motohiro14fec792010-08-10 18:03:05 -07001429 noswap, get_swappiness(victim), zone);
Balbir Singh4e416952009-09-23 15:56:39 -07001430 else
1431 ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
1432 noswap, get_swappiness(victim));
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001433 css_put(&victim->css);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001434 /*
1435 * At shrinking usage, we can't check we should stop here or
1436 * reclaim more. It's depends on callers. last_scanned_child
1437 * will work enough for keeping fairness under tree.
1438 */
1439 if (shrink)
1440 return ret;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001441 total += ret;
Balbir Singh4e416952009-09-23 15:56:39 -07001442 if (check_soft) {
1443 if (res_counter_check_under_soft_limit(&root_mem->res))
1444 return total;
1445 } else if (mem_cgroup_check_under_limit(root_mem))
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001446 return 1 + total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001447 }
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001448 return total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001449}
1450
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001451/*
1452 * Check OOM-Killer is already running under our hierarchy.
1453 * If someone is running, return false.
1454 */
1455static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
1456{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001457 int x, lock_count = 0;
1458 struct mem_cgroup *iter;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001459
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001460 for_each_mem_cgroup_tree(iter, mem) {
1461 x = atomic_inc_return(&iter->oom_lock);
1462 lock_count = max(x, lock_count);
1463 }
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001464
1465 if (lock_count == 1)
1466 return true;
1467 return false;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001468}
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001469
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001470static int mem_cgroup_oom_unlock(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001471{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001472 struct mem_cgroup *iter;
1473
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001474 /*
1475 * When a new child is created while the hierarchy is under oom,
1476 * mem_cgroup_oom_lock() may not be called. We have to use
1477 * atomic_add_unless() here.
1478 */
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001479 for_each_mem_cgroup_tree(iter, mem)
1480 atomic_add_unless(&iter->oom_lock, -1, 0);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001481 return 0;
1482}
1483
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001484
1485static DEFINE_MUTEX(memcg_oom_mutex);
1486static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1487
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001488struct oom_wait_info {
1489 struct mem_cgroup *mem;
1490 wait_queue_t wait;
1491};
1492
1493static int memcg_oom_wake_function(wait_queue_t *wait,
1494 unsigned mode, int sync, void *arg)
1495{
1496 struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg;
1497 struct oom_wait_info *oom_wait_info;
1498
1499 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1500
1501 if (oom_wait_info->mem == wake_mem)
1502 goto wakeup;
1503 /* if no hierarchy, no match */
1504 if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy)
1505 return 0;
1506 /*
1507 * Both of oom_wait_info->mem and wake_mem are stable under us.
1508 * Then we can use css_is_ancestor without taking care of RCU.
1509 */
1510 if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) &&
1511 !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css))
1512 return 0;
1513
1514wakeup:
1515 return autoremove_wake_function(wait, mode, sync, arg);
1516}
1517
1518static void memcg_wakeup_oom(struct mem_cgroup *mem)
1519{
1520 /* for filtering, pass "mem" as argument. */
1521 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem);
1522}
1523
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001524static void memcg_oom_recover(struct mem_cgroup *mem)
1525{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001526 if (mem && atomic_read(&mem->oom_lock))
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001527 memcg_wakeup_oom(mem);
1528}
1529
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001530/*
1531 * try to call OOM killer. returns false if we should exit memory-reclaim loop.
1532 */
1533bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
1534{
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001535 struct oom_wait_info owait;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001536 bool locked, need_to_kill;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001537
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001538 owait.mem = mem;
1539 owait.wait.flags = 0;
1540 owait.wait.func = memcg_oom_wake_function;
1541 owait.wait.private = current;
1542 INIT_LIST_HEAD(&owait.wait.task_list);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001543 need_to_kill = true;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001544 /* At first, try to OOM lock hierarchy under mem.*/
1545 mutex_lock(&memcg_oom_mutex);
1546 locked = mem_cgroup_oom_lock(mem);
1547 /*
1548 * Even if signal_pending(), we can't quit charge() loop without
1549 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
1550 * under OOM is always welcomed, use TASK_KILLABLE here.
1551 */
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001552 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1553 if (!locked || mem->oom_kill_disable)
1554 need_to_kill = false;
1555 if (locked)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07001556 mem_cgroup_oom_notify(mem);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001557 mutex_unlock(&memcg_oom_mutex);
1558
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001559 if (need_to_kill) {
1560 finish_wait(&memcg_oom_waitq, &owait.wait);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001561 mem_cgroup_out_of_memory(mem, mask);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001562 } else {
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001563 schedule();
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001564 finish_wait(&memcg_oom_waitq, &owait.wait);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001565 }
1566 mutex_lock(&memcg_oom_mutex);
1567 mem_cgroup_oom_unlock(mem);
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001568 memcg_wakeup_oom(mem);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001569 mutex_unlock(&memcg_oom_mutex);
1570
1571 if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
1572 return false;
1573 /* Give chance to dying process */
1574 schedule_timeout(1);
1575 return true;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001576}
1577
Balbir Singhd69b0422009-06-17 16:26:34 -07001578/*
1579 * Currently used to update mapped file statistics, but the routine can be
1580 * generalized to update other statistics as well.
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001581 *
1582 * Notes: Race condition
1583 *
1584 * We usually use page_cgroup_lock() for accessing page_cgroup member but
1585 * it tends to be costly. But considering some conditions, we doesn't need
1586 * to do so _always_.
1587 *
1588 * Considering "charge", lock_page_cgroup() is not required because all
1589 * file-stat operations happen after a page is attached to radix-tree. There
1590 * are no race with "charge".
1591 *
1592 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
1593 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
1594 * if there are race with "uncharge". Statistics itself is properly handled
1595 * by flags.
1596 *
1597 * Considering "move", this is an only case we see a race. To make the race
1598 * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are
1599 * possibility of race condition. If there is, we take a lock.
Balbir Singhd69b0422009-06-17 16:26:34 -07001600 */
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07001601
Greg Thelen2a7106f2011-01-13 15:47:37 -08001602void mem_cgroup_update_page_stat(struct page *page,
1603 enum mem_cgroup_page_stat_item idx, int val)
Balbir Singhd69b0422009-06-17 16:26:34 -07001604{
1605 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001606 struct page_cgroup *pc = lookup_page_cgroup(page);
1607 bool need_unlock = false;
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08001608 unsigned long uninitialized_var(flags);
Balbir Singhd69b0422009-06-17 16:26:34 -07001609
Balbir Singhd69b0422009-06-17 16:26:34 -07001610 if (unlikely(!pc))
1611 return;
1612
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001613 rcu_read_lock();
Balbir Singhd69b0422009-06-17 16:26:34 -07001614 mem = pc->mem_cgroup;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001615 if (unlikely(!mem || !PageCgroupUsed(pc)))
1616 goto out;
1617 /* pc->mem_cgroup is unstable ? */
1618 if (unlikely(mem_cgroup_stealed(mem))) {
1619 /* take a lock against to access pc->mem_cgroup */
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08001620 move_lock_page_cgroup(pc, &flags);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001621 need_unlock = true;
1622 mem = pc->mem_cgroup;
1623 if (!mem || !PageCgroupUsed(pc))
1624 goto out;
1625 }
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07001626
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07001627 switch (idx) {
Greg Thelen2a7106f2011-01-13 15:47:37 -08001628 case MEMCG_NR_FILE_MAPPED:
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07001629 if (val > 0)
1630 SetPageCgroupFileMapped(pc);
1631 else if (!page_mapped(page))
KAMEZAWA Hiroyuki0c270f82010-10-27 15:33:39 -07001632 ClearPageCgroupFileMapped(pc);
Greg Thelen2a7106f2011-01-13 15:47:37 -08001633 idx = MEM_CGROUP_STAT_FILE_MAPPED;
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07001634 break;
1635 default:
1636 BUG();
KAMEZAWA Hiroyuki8725d542010-04-06 14:35:05 -07001637 }
Balbir Singhd69b0422009-06-17 16:26:34 -07001638
Greg Thelen2a7106f2011-01-13 15:47:37 -08001639 this_cpu_add(mem->stat->count[idx], val);
1640
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001641out:
1642 if (unlikely(need_unlock))
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08001643 move_unlock_page_cgroup(pc, &flags);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001644 rcu_read_unlock();
1645 return;
Balbir Singhd69b0422009-06-17 16:26:34 -07001646}
Greg Thelen2a7106f2011-01-13 15:47:37 -08001647EXPORT_SYMBOL(mem_cgroup_update_page_stat);
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07001648
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001649/*
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001650 * size of first charge trial. "32" comes from vmscan.c's magic value.
1651 * TODO: maybe necessary to use big numbers in big irons.
1652 */
1653#define CHARGE_SIZE (32 * PAGE_SIZE)
1654struct memcg_stock_pcp {
1655 struct mem_cgroup *cached; /* this never be root cgroup */
1656 int charge;
1657 struct work_struct work;
1658};
1659static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1660static atomic_t memcg_drain_count;
1661
1662/*
1663 * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed
1664 * from local stock and true is returned. If the stock is 0 or charges from a
1665 * cgroup which is not current target, returns false. This stock will be
1666 * refilled.
1667 */
1668static bool consume_stock(struct mem_cgroup *mem)
1669{
1670 struct memcg_stock_pcp *stock;
1671 bool ret = true;
1672
1673 stock = &get_cpu_var(memcg_stock);
1674 if (mem == stock->cached && stock->charge)
1675 stock->charge -= PAGE_SIZE;
1676 else /* need to call res_counter_charge */
1677 ret = false;
1678 put_cpu_var(memcg_stock);
1679 return ret;
1680}
1681
1682/*
1683 * Returns stocks cached in percpu to res_counter and reset cached information.
1684 */
1685static void drain_stock(struct memcg_stock_pcp *stock)
1686{
1687 struct mem_cgroup *old = stock->cached;
1688
1689 if (stock->charge) {
1690 res_counter_uncharge(&old->res, stock->charge);
1691 if (do_swap_account)
1692 res_counter_uncharge(&old->memsw, stock->charge);
1693 }
1694 stock->cached = NULL;
1695 stock->charge = 0;
1696}
1697
1698/*
1699 * This must be called under preempt disabled or must be called by
1700 * a thread which is pinned to local cpu.
1701 */
1702static void drain_local_stock(struct work_struct *dummy)
1703{
1704 struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
1705 drain_stock(stock);
1706}
1707
1708/*
1709 * Cache charges(val) which is from res_counter, to local per_cpu area.
Greg Thelen320cc512010-03-15 15:27:28 +01001710 * This will be consumed by consume_stock() function, later.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001711 */
1712static void refill_stock(struct mem_cgroup *mem, int val)
1713{
1714 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
1715
1716 if (stock->cached != mem) { /* reset if necessary */
1717 drain_stock(stock);
1718 stock->cached = mem;
1719 }
1720 stock->charge += val;
1721 put_cpu_var(memcg_stock);
1722}
1723
1724/*
1725 * Tries to drain stocked charges in other cpus. This function is asynchronous
1726 * and just put a work per cpu for draining localy on each cpu. Caller can
1727 * expects some charges will be back to res_counter later but cannot wait for
1728 * it.
1729 */
1730static void drain_all_stock_async(void)
1731{
1732 int cpu;
1733 /* This function is for scheduling "drain" in asynchronous way.
1734 * The result of "drain" is not directly handled by callers. Then,
1735 * if someone is calling drain, we don't have to call drain more.
1736 * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if
1737 * there is a race. We just do loose check here.
1738 */
1739 if (atomic_read(&memcg_drain_count))
1740 return;
1741 /* Notify other cpus that system-wide "drain" is running */
1742 atomic_inc(&memcg_drain_count);
1743 get_online_cpus();
1744 for_each_online_cpu(cpu) {
1745 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1746 schedule_work_on(cpu, &stock->work);
1747 }
1748 put_online_cpus();
1749 atomic_dec(&memcg_drain_count);
1750 /* We don't wait for flush_work */
1751}
1752
1753/* This is a synchronous drain interface. */
1754static void drain_all_stock_sync(void)
1755{
1756 /* called when force_empty is called */
1757 atomic_inc(&memcg_drain_count);
1758 schedule_on_each_cpu(drain_local_stock);
1759 atomic_dec(&memcg_drain_count);
1760}
1761
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001762/*
1763 * This function drains percpu counter value from DEAD cpu and
1764 * move it to local cpu. Note that this function can be preempted.
1765 */
1766static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu)
1767{
1768 int i;
1769
1770 spin_lock(&mem->pcp_counter_lock);
1771 for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) {
1772 s64 x = per_cpu(mem->stat->count[i], cpu);
1773
1774 per_cpu(mem->stat->count[i], cpu) = 0;
1775 mem->nocpu_base.count[i] += x;
1776 }
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001777 /* need to clear ON_MOVE value, works as a kind of lock. */
1778 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
1779 spin_unlock(&mem->pcp_counter_lock);
1780}
1781
1782static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu)
1783{
1784 int idx = MEM_CGROUP_ON_MOVE;
1785
1786 spin_lock(&mem->pcp_counter_lock);
1787 per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx];
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001788 spin_unlock(&mem->pcp_counter_lock);
1789}
1790
1791static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001792 unsigned long action,
1793 void *hcpu)
1794{
1795 int cpu = (unsigned long)hcpu;
1796 struct memcg_stock_pcp *stock;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001797 struct mem_cgroup *iter;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001798
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001799 if ((action == CPU_ONLINE)) {
1800 for_each_mem_cgroup_all(iter)
1801 synchronize_mem_cgroup_on_move(iter, cpu);
1802 return NOTIFY_OK;
1803 }
1804
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001805 if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001806 return NOTIFY_OK;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001807
1808 for_each_mem_cgroup_all(iter)
1809 mem_cgroup_drain_pcp_counter(iter, cpu);
1810
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001811 stock = &per_cpu(memcg_stock, cpu);
1812 drain_stock(stock);
1813 return NOTIFY_OK;
1814}
1815
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001816
1817/* See __mem_cgroup_try_charge() for details */
1818enum {
1819 CHARGE_OK, /* success */
1820 CHARGE_RETRY, /* need to retry but retry is not bad */
1821 CHARGE_NOMEM, /* we can't do more. return -ENOMEM */
1822 CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */
1823 CHARGE_OOM_DIE, /* the current is killed because of OOM */
1824};
1825
1826static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
1827 int csize, bool oom_check)
1828{
1829 struct mem_cgroup *mem_over_limit;
1830 struct res_counter *fail_res;
1831 unsigned long flags = 0;
1832 int ret;
1833
1834 ret = res_counter_charge(&mem->res, csize, &fail_res);
1835
1836 if (likely(!ret)) {
1837 if (!do_swap_account)
1838 return CHARGE_OK;
1839 ret = res_counter_charge(&mem->memsw, csize, &fail_res);
1840 if (likely(!ret))
1841 return CHARGE_OK;
1842
1843 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
1844 flags |= MEM_CGROUP_RECLAIM_NOSWAP;
1845 } else
1846 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
1847
1848 if (csize > PAGE_SIZE) /* change csize and retry */
1849 return CHARGE_RETRY;
1850
1851 if (!(gfp_mask & __GFP_WAIT))
1852 return CHARGE_WOULDBLOCK;
1853
1854 ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
1855 gfp_mask, flags);
1856 /*
1857 * try_to_free_mem_cgroup_pages() might not give us a full
1858 * picture of reclaim. Some pages are reclaimed and might be
1859 * moved to swap cache or just unmapped from the cgroup.
1860 * Check the limit again to see if the reclaim reduced the
1861 * current usage of the cgroup before giving up
1862 */
1863 if (ret || mem_cgroup_check_under_limit(mem_over_limit))
1864 return CHARGE_RETRY;
1865
1866 /*
1867 * At task move, charge accounts can be doubly counted. So, it's
1868 * better to wait until the end of task_move if something is going on.
1869 */
1870 if (mem_cgroup_wait_acct_move(mem_over_limit))
1871 return CHARGE_RETRY;
1872
1873 /* If we don't need to call oom-killer at el, return immediately */
1874 if (!oom_check)
1875 return CHARGE_NOMEM;
1876 /* check OOM */
1877 if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask))
1878 return CHARGE_OOM_DIE;
1879
1880 return CHARGE_RETRY;
1881}
1882
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001883/*
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001884 * Unlike exported interface, "oom" parameter is added. if oom==true,
1885 * oom-killer can be invoked.
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001886 */
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08001887static int __mem_cgroup_try_charge(struct mm_struct *mm,
Andrea Arcangeliec168512011-01-13 15:46:56 -08001888 gfp_t gfp_mask,
1889 struct mem_cgroup **memcg, bool oom,
1890 int page_size)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001891{
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001892 int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
1893 struct mem_cgroup *mem = NULL;
1894 int ret;
Andrea Arcangeliec168512011-01-13 15:46:56 -08001895 int csize = max(CHARGE_SIZE, (unsigned long) page_size);
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001896
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001897 /*
1898 * Unlike gloval-vm's OOM-kill, we're not in memory shortage
1899 * in system level. So, allow to go ahead dying process in addition to
1900 * MEMDIE process.
1901 */
1902 if (unlikely(test_thread_flag(TIF_MEMDIE)
1903 || fatal_signal_pending(current)))
1904 goto bypass;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001905
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001906 /*
Hugh Dickins3be91272008-02-07 00:14:19 -08001907 * We always charge the cgroup the mm_struct belongs to.
1908 * The mm_struct's mem_cgroup changes on task migration if the
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001909 * thread group leader migrates. It's possible that mm is not
1910 * set, if so charge the init_mm (happens for pagecache usage).
1911 */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001912 if (!*memcg && !mm)
1913 goto bypass;
1914again:
1915 if (*memcg) { /* css should be a valid one */
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001916 mem = *memcg;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001917 VM_BUG_ON(css_is_removed(&mem->css));
1918 if (mem_cgroup_is_root(mem))
1919 goto done;
Andrea Arcangeliec168512011-01-13 15:46:56 -08001920 if (page_size == PAGE_SIZE && consume_stock(mem))
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001921 goto done;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001922 css_get(&mem->css);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001923 } else {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001924 struct task_struct *p;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001925
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001926 rcu_read_lock();
1927 p = rcu_dereference(mm->owner);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001928 /*
KAMEZAWA Hiroyukiebb76ce2010-12-29 14:07:11 -08001929 * Because we don't have task_lock(), "p" can exit.
1930 * In that case, "mem" can point to root or p can be NULL with
1931 * race with swapoff. Then, we have small risk of mis-accouning.
1932 * But such kind of mis-account by race always happens because
1933 * we don't have cgroup_mutex(). It's overkill and we allo that
1934 * small race, here.
1935 * (*) swapoff at el will charge against mm-struct not against
1936 * task-struct. So, mm->owner can be NULL.
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001937 */
1938 mem = mem_cgroup_from_task(p);
KAMEZAWA Hiroyukiebb76ce2010-12-29 14:07:11 -08001939 if (!mem || mem_cgroup_is_root(mem)) {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001940 rcu_read_unlock();
1941 goto done;
1942 }
Andrea Arcangeliec168512011-01-13 15:46:56 -08001943 if (page_size == PAGE_SIZE && consume_stock(mem)) {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001944 /*
1945 * It seems dagerous to access memcg without css_get().
1946 * But considering how consume_stok works, it's not
1947 * necessary. If consume_stock success, some charges
1948 * from this memcg are cached on this cpu. So, we
1949 * don't need to call css_get()/css_tryget() before
1950 * calling consume_stock().
1951 */
1952 rcu_read_unlock();
1953 goto done;
1954 }
1955 /* after here, we may be blocked. we need to get refcnt */
1956 if (!css_tryget(&mem->css)) {
1957 rcu_read_unlock();
1958 goto again;
1959 }
1960 rcu_read_unlock();
1961 }
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08001962
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001963 do {
1964 bool oom_check;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001965
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001966 /* If killed, bypass charge */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001967 if (fatal_signal_pending(current)) {
1968 css_put(&mem->css);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001969 goto bypass;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001970 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08001971
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001972 oom_check = false;
1973 if (oom && !nr_oom_retries) {
1974 oom_check = true;
1975 nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
1976 }
Balbir Singh6d61ef42009-01-07 18:08:06 -08001977
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001978 ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check);
1979
1980 switch (ret) {
1981 case CHARGE_OK:
1982 break;
1983 case CHARGE_RETRY: /* not in OOM situation but retry */
Andrea Arcangeliec168512011-01-13 15:46:56 -08001984 csize = page_size;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001985 css_put(&mem->css);
1986 mem = NULL;
1987 goto again;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001988 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001989 css_put(&mem->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08001990 goto nomem;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001991 case CHARGE_NOMEM: /* OOM routine works */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001992 if (!oom) {
1993 css_put(&mem->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001994 goto nomem;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07001995 }
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001996 /* If oom, we never return -ENOMEM */
1997 nr_oom_retries--;
1998 break;
1999 case CHARGE_OOM_DIE: /* Killed by OOM Killer */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002000 css_put(&mem->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002001 goto bypass;
Balbir Singh66e17072008-02-07 00:13:56 -08002002 }
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002003 } while (ret != CHARGE_OK);
2004
Andrea Arcangeliec168512011-01-13 15:46:56 -08002005 if (csize > page_size)
2006 refill_stock(mem, csize - page_size);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002007 css_put(&mem->css);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002008done:
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002009 *memcg = mem;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002010 return 0;
2011nomem:
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002012 *memcg = NULL;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002013 return -ENOMEM;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002014bypass:
2015 *memcg = NULL;
2016 return 0;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002017}
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002018
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002019/*
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002020 * Somemtimes we have to undo a charge we got by try_charge().
2021 * This function is for that and do uncharge, put css's refcnt.
2022 * gotten by try_charge().
2023 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002024static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
2025 unsigned long count)
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002026{
2027 if (!mem_cgroup_is_root(mem)) {
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002028 res_counter_uncharge(&mem->res, PAGE_SIZE * count);
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002029 if (do_swap_account)
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002030 res_counter_uncharge(&mem->memsw, PAGE_SIZE * count);
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002031 }
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002032}
2033
Andrea Arcangeliec168512011-01-13 15:46:56 -08002034static void mem_cgroup_cancel_charge(struct mem_cgroup *mem,
2035 int page_size)
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002036{
Andrea Arcangeliec168512011-01-13 15:46:56 -08002037 __mem_cgroup_cancel_charge(mem, page_size >> PAGE_SHIFT);
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002038}
2039
2040/*
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002041 * A helper function to get mem_cgroup from ID. must be called under
2042 * rcu_read_lock(). The caller must check css_is_removed() or some if
2043 * it's concern. (dropping refcnt from swap can be called against removed
2044 * memcg.)
2045 */
2046static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
2047{
2048 struct cgroup_subsys_state *css;
2049
2050 /* ID 0 is unused ID */
2051 if (!id)
2052 return NULL;
2053 css = css_lookup(&mem_cgroup_subsys, id);
2054 if (!css)
2055 return NULL;
2056 return container_of(css, struct mem_cgroup, css);
2057}
2058
Wu Fengguange42d9d52009-12-16 12:19:59 +01002059struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002060{
Wu Fengguange42d9d52009-12-16 12:19:59 +01002061 struct mem_cgroup *mem = NULL;
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002062 struct page_cgroup *pc;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002063 unsigned short id;
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002064 swp_entry_t ent;
2065
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002066 VM_BUG_ON(!PageLocked(page));
2067
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002068 pc = lookup_page_cgroup(page);
Daisuke Nishimurac0bd3f62009-04-30 15:08:11 -07002069 lock_page_cgroup(pc);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002070 if (PageCgroupUsed(pc)) {
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002071 mem = pc->mem_cgroup;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002072 if (mem && !css_tryget(&mem->css))
2073 mem = NULL;
Wu Fengguange42d9d52009-12-16 12:19:59 +01002074 } else if (PageSwapCache(page)) {
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002075 ent.val = page_private(page);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002076 id = lookup_swap_cgroup(ent);
2077 rcu_read_lock();
2078 mem = mem_cgroup_lookup(id);
2079 if (mem && !css_tryget(&mem->css))
2080 mem = NULL;
2081 rcu_read_unlock();
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002082 }
Daisuke Nishimurac0bd3f62009-04-30 15:08:11 -07002083 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002084 return mem;
2085}
2086
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002087/*
Daisuke Nishimuraa5e924f2009-01-07 18:08:28 -08002088 * commit a charge got by __mem_cgroup_try_charge() and makes page_cgroup to be
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002089 * USED state. If already USED, uncharge and return.
2090 */
Daisuke Nishimura152c9cc2011-01-13 15:46:56 -08002091static void ____mem_cgroup_commit_charge(struct mem_cgroup *mem,
2092 struct page_cgroup *pc,
2093 enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002094{
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002095 pc->mem_cgroup = mem;
KAMEZAWA Hiroyuki261fb612009-09-23 15:56:33 -07002096 /*
2097 * We access a page_cgroup asynchronously without lock_page_cgroup().
2098 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
2099 * is accessed after testing USED bit. To make pc->mem_cgroup visible
2100 * before USED bit, we need memory barrier here.
2101 * See mem_cgroup_add_lru_list(), etc.
2102 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002103 smp_wmb();
Balbir Singh4b3bde42009-09-23 15:56:32 -07002104 switch (ctype) {
2105 case MEM_CGROUP_CHARGE_TYPE_CACHE:
2106 case MEM_CGROUP_CHARGE_TYPE_SHMEM:
2107 SetPageCgroupCache(pc);
2108 SetPageCgroupUsed(pc);
2109 break;
2110 case MEM_CGROUP_CHARGE_TYPE_MAPPED:
2111 ClearPageCgroupCache(pc);
2112 SetPageCgroupUsed(pc);
2113 break;
2114 default:
2115 break;
2116 }
Hugh Dickins3be91272008-02-07 00:14:19 -08002117
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002118 mem_cgroup_charge_statistics(mem, pc, true);
Daisuke Nishimura152c9cc2011-01-13 15:46:56 -08002119}
2120
2121static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
2122 struct page_cgroup *pc,
2123 enum charge_type ctype,
2124 int page_size)
2125{
2126 int i;
2127 int count = page_size >> PAGE_SHIFT;
2128
2129 /* try_charge() can return NULL to *memcg, taking care of it. */
2130 if (!mem)
2131 return;
2132
2133 lock_page_cgroup(pc);
2134 if (unlikely(PageCgroupUsed(pc))) {
2135 unlock_page_cgroup(pc);
2136 mem_cgroup_cancel_charge(mem, page_size);
2137 return;
2138 }
2139
2140 /*
2141 * we don't need page_cgroup_lock about tail pages, becase they are not
2142 * accessed by any other context at this point.
2143 */
2144 for (i = 0; i < count; i++)
2145 ____mem_cgroup_commit_charge(mem, pc + i, ctype);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002146
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002147 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki430e48632010-03-10 15:22:30 -08002148 /*
2149 * "charge_statistics" updated event counter. Then, check it.
2150 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
2151 * if they exceeds softlimit.
2152 */
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08002153 memcg_check_events(mem, pc->page);
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002154}
2155
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002156/**
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002157 * __mem_cgroup_move_account - move account of the page
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002158 * @pc: page_cgroup of the page.
2159 * @from: mem_cgroup which the page is moved from.
2160 * @to: mem_cgroup which the page is moved to. @from != @to.
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002161 * @uncharge: whether we should call uncharge and css_put against @from.
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002162 *
2163 * The caller must confirm following.
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002164 * - page is not on LRU (isolate_page() is useful.)
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002165 * - the pc is locked, used, and ->mem_cgroup points to @from.
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002166 *
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002167 * This function doesn't do "charge" nor css_get to new cgroup. It should be
2168 * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is
2169 * true, this function does "uncharge" from old cgroup, but it doesn't if
2170 * @uncharge is false, so a caller should do "uncharge".
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002171 */
2172
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002173static void __mem_cgroup_move_account(struct page_cgroup *pc,
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002174 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002175{
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002176 VM_BUG_ON(from == to);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002177 VM_BUG_ON(PageLRU(pc->page));
Kirill A. Shutemov112bc2e2010-11-24 12:56:58 -08002178 VM_BUG_ON(!page_is_cgroup_locked(pc));
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002179 VM_BUG_ON(!PageCgroupUsed(pc));
2180 VM_BUG_ON(pc->mem_cgroup != from);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002181
KAMEZAWA Hiroyuki8725d542010-04-06 14:35:05 -07002182 if (PageCgroupFileMapped(pc)) {
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08002183 /* Update mapped_file data for mem_cgroup */
2184 preempt_disable();
2185 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
2186 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
2187 preempt_enable();
Balbir Singhd69b0422009-06-17 16:26:34 -07002188 }
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002189 mem_cgroup_charge_statistics(from, pc, false);
2190 if (uncharge)
2191 /* This is not "cancel", but cancel_charge does all we need. */
Andrea Arcangeliec168512011-01-13 15:46:56 -08002192 mem_cgroup_cancel_charge(from, PAGE_SIZE);
Balbir Singhd69b0422009-06-17 16:26:34 -07002193
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002194 /* caller should have done css_get */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002195 pc->mem_cgroup = to;
2196 mem_cgroup_charge_statistics(to, pc, true);
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002197 /*
2198 * We charges against "to" which may not have any tasks. Then, "to"
2199 * can be under rmdir(). But in current implementation, caller of
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08002200 * this function is just force_empty() and move charge, so it's
2201 * garanteed that "to" is never removed. So, we don't check rmdir
2202 * status here.
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002203 */
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002204}
2205
2206/*
2207 * check whether the @pc is valid for moving account and call
2208 * __mem_cgroup_move_account()
2209 */
2210static int mem_cgroup_move_account(struct page_cgroup *pc,
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002211 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002212{
2213 int ret = -EINVAL;
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08002214 unsigned long flags;
2215
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002216 lock_page_cgroup(pc);
2217 if (PageCgroupUsed(pc) && pc->mem_cgroup == from) {
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08002218 move_lock_page_cgroup(pc, &flags);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002219 __mem_cgroup_move_account(pc, from, to, uncharge);
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08002220 move_unlock_page_cgroup(pc, &flags);
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002221 ret = 0;
2222 }
2223 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08002224 /*
2225 * check events
2226 */
2227 memcg_check_events(to, pc->page);
2228 memcg_check_events(from, pc->page);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002229 return ret;
2230}
2231
2232/*
2233 * move charges to its parent.
2234 */
2235
2236static int mem_cgroup_move_parent(struct page_cgroup *pc,
2237 struct mem_cgroup *child,
2238 gfp_t gfp_mask)
2239{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002240 struct page *page = pc->page;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002241 struct cgroup *cg = child->css.cgroup;
2242 struct cgroup *pcg = cg->parent;
2243 struct mem_cgroup *parent;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002244 int ret;
2245
2246 /* Is ROOT ? */
2247 if (!pcg)
2248 return -EINVAL;
2249
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002250 ret = -EBUSY;
2251 if (!get_page_unless_zero(page))
2252 goto out;
2253 if (isolate_lru_page(page))
2254 goto put;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002255
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002256 parent = mem_cgroup_from_cont(pcg);
Andrea Arcangeliec168512011-01-13 15:46:56 -08002257 ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false,
2258 PAGE_SIZE);
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002259 if (ret || !parent)
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002260 goto put_back;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002261
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002262 ret = mem_cgroup_move_account(pc, child, parent, true);
2263 if (ret)
Andrea Arcangeliec168512011-01-13 15:46:56 -08002264 mem_cgroup_cancel_charge(parent, PAGE_SIZE);
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002265put_back:
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002266 putback_lru_page(page);
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002267put:
Daisuke Nishimura40d58132009-01-15 13:51:12 -08002268 put_page(page);
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08002269out:
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002270 return ret;
2271}
2272
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002273/*
2274 * Charge the memory controller for page usage.
2275 * Return
2276 * 0 if the charge was successful
2277 * < 0 if the cgroup is over its limit
2278 */
2279static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002280 gfp_t gfp_mask, enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002281{
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002282 struct mem_cgroup *mem = NULL;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002283 struct page_cgroup *pc;
2284 int ret;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002285 int page_size = PAGE_SIZE;
2286
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08002287 if (PageTransHuge(page)) {
Andrea Arcangeliec168512011-01-13 15:46:56 -08002288 page_size <<= compound_order(page);
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08002289 VM_BUG_ON(!PageTransHuge(page));
2290 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002291
2292 pc = lookup_page_cgroup(page);
2293 /* can happen at boot */
2294 if (unlikely(!pc))
2295 return 0;
2296 prefetchw(pc);
2297
Andrea Arcangeliec168512011-01-13 15:46:56 -08002298 ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true, page_size);
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002299 if (ret || !mem)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002300 return ret;
2301
Andrea Arcangeliec168512011-01-13 15:46:56 -08002302 __mem_cgroup_commit_charge(mem, pc, ctype, page_size);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002303 return 0;
2304}
2305
2306int mem_cgroup_newpage_charge(struct page *page,
2307 struct mm_struct *mm, gfp_t gfp_mask)
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08002308{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002309 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07002310 return 0;
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002311 /*
2312 * If already mapped, we don't have to account.
2313 * If page cache, page->mapping has address_space.
2314 * But page->mapping may have out-of-use anon_vma pointer,
2315 * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
2316 * is NULL.
2317 */
2318 if (page_mapped(page) || (page->mapping && !PageAnon(page)))
2319 return 0;
2320 if (unlikely(!mm))
2321 mm = &init_mm;
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08002322 return mem_cgroup_charge_common(page, mm, gfp_mask,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002323 MEM_CGROUP_CHARGE_TYPE_MAPPED);
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08002324}
2325
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002326static void
2327__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2328 enum charge_type ctype);
2329
Balbir Singhe1a1cd52008-02-07 00:14:02 -08002330int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
2331 gfp_t gfp_mask)
Balbir Singh8697d332008-02-07 00:13:59 -08002332{
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002333 int ret;
2334
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002335 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07002336 return 0;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002337 if (PageCompound(page))
2338 return 0;
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002339 /*
2340 * Corner case handling. This is called from add_to_page_cache()
2341 * in usual. But some FS (shmem) precharges this page before calling it
2342 * and call add_to_page_cache() with GFP_NOWAIT.
2343 *
2344 * For GFP_NOWAIT case, the page may be pre-charged before calling
2345 * add_to_page_cache(). (See shmem.c) check it here and avoid to call
2346 * charge twice. (It works but has to pay a bit larger cost.)
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002347 * And when the page is SwapCache, it should take swap information
2348 * into account. This is under lock_page() now.
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002349 */
2350 if (!(gfp_mask & __GFP_WAIT)) {
2351 struct page_cgroup *pc;
2352
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002353 pc = lookup_page_cgroup(page);
2354 if (!pc)
2355 return 0;
2356 lock_page_cgroup(pc);
2357 if (PageCgroupUsed(pc)) {
2358 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002359 return 0;
2360 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002361 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002362 }
2363
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002364 if (unlikely(!mm))
Balbir Singh8697d332008-02-07 00:13:59 -08002365 mm = &init_mm;
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002366
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -07002367 if (page_is_file_cache(page))
2368 return mem_cgroup_charge_common(page, mm, gfp_mask,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002369 MEM_CGROUP_CHARGE_TYPE_CACHE);
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002370
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002371 /* shmem */
2372 if (PageSwapCache(page)) {
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002373 struct mem_cgroup *mem = NULL;
2374
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002375 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2376 if (!ret)
2377 __mem_cgroup_commit_charge_swapin(page, mem,
2378 MEM_CGROUP_CHARGE_TYPE_SHMEM);
2379 } else
2380 ret = mem_cgroup_charge_common(page, mm, gfp_mask,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002381 MEM_CGROUP_CHARGE_TYPE_SHMEM);
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002382
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002383 return ret;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002384}
2385
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002386/*
2387 * While swap-in, try_charge -> commit or cancel, the page is locked.
2388 * And when try_charge() successfully returns, one refcnt to memcg without
Uwe Kleine-König21ae2952009-10-07 15:21:09 +02002389 * struct page_cgroup is acquired. This refcnt will be consumed by
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002390 * "commit()" or removed by "cancel()"
2391 */
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002392int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
2393 struct page *page,
2394 gfp_t mask, struct mem_cgroup **ptr)
2395{
2396 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002397 int ret;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002398
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002399 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002400 return 0;
2401
2402 if (!do_swap_account)
2403 goto charge_cur_mm;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002404 /*
2405 * A racing thread's fault, or swapoff, may have already updated
Hugh Dickins407f9c82009-12-14 17:59:30 -08002406 * the pte, and even removed page from swap cache: in those cases
2407 * do_swap_page()'s pte_same() test will fail; but there's also a
2408 * KSM case which does need to charge the page.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002409 */
2410 if (!PageSwapCache(page))
Hugh Dickins407f9c82009-12-14 17:59:30 -08002411 goto charge_cur_mm;
Wu Fengguange42d9d52009-12-16 12:19:59 +01002412 mem = try_get_mem_cgroup_from_page(page);
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002413 if (!mem)
2414 goto charge_cur_mm;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002415 *ptr = mem;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002416 ret = __mem_cgroup_try_charge(NULL, mask, ptr, true, PAGE_SIZE);
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002417 css_put(&mem->css);
2418 return ret;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002419charge_cur_mm:
2420 if (unlikely(!mm))
2421 mm = &init_mm;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002422 return __mem_cgroup_try_charge(mm, mask, ptr, true, PAGE_SIZE);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002423}
2424
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002425static void
2426__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2427 enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002428{
2429 struct page_cgroup *pc;
2430
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002431 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002432 return;
2433 if (!ptr)
2434 return;
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002435 cgroup_exclude_rmdir(&ptr->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002436 pc = lookup_page_cgroup(page);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08002437 mem_cgroup_lru_del_before_commit_swapcache(page);
Andrea Arcangeliec168512011-01-13 15:46:56 -08002438 __mem_cgroup_commit_charge(ptr, pc, ctype, PAGE_SIZE);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08002439 mem_cgroup_lru_add_after_commit_swapcache(page);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002440 /*
2441 * Now swap is on-memory. This means this page may be
2442 * counted both as mem and swap....double count.
KAMEZAWA Hiroyuki03f3c432009-01-07 18:08:31 -08002443 * Fix it by uncharging from memsw. Basically, this SwapCache is stable
2444 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
2445 * may call delete_from_swap_cache() before reach here.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002446 */
KAMEZAWA Hiroyuki03f3c432009-01-07 18:08:31 -08002447 if (do_swap_account && PageSwapCache(page)) {
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002448 swp_entry_t ent = {.val = page_private(page)};
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002449 unsigned short id;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002450 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002451
2452 id = swap_cgroup_record(ent, 0);
2453 rcu_read_lock();
2454 memcg = mem_cgroup_lookup(id);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002455 if (memcg) {
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002456 /*
2457 * This recorded memcg can be obsolete one. So, avoid
2458 * calling css_tryget
2459 */
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002460 if (!mem_cgroup_is_root(memcg))
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -07002461 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002462 mem_cgroup_swap_statistics(memcg, false);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002463 mem_cgroup_put(memcg);
2464 }
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002465 rcu_read_unlock();
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002466 }
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002467 /*
2468 * At swapin, we may charge account against cgroup which has no tasks.
2469 * So, rmdir()->pre_destroy() can be called while we do this charge.
2470 * In that case, we need to call pre_destroy() again. check it here.
2471 */
2472 cgroup_release_and_wakeup_rmdir(&ptr->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002473}
2474
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002475void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
2476{
2477 __mem_cgroup_commit_charge_swapin(page, ptr,
2478 MEM_CGROUP_CHARGE_TYPE_MAPPED);
2479}
2480
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002481void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
2482{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002483 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002484 return;
2485 if (!mem)
2486 return;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002487 mem_cgroup_cancel_charge(mem, PAGE_SIZE);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002488}
2489
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002490static void
Andrea Arcangeliec168512011-01-13 15:46:56 -08002491__do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype,
2492 int page_size)
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002493{
2494 struct memcg_batch_info *batch = NULL;
2495 bool uncharge_memsw = true;
2496 /* If swapout, usage of swap doesn't decrease */
2497 if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
2498 uncharge_memsw = false;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002499
2500 batch = &current->memcg_batch;
2501 /*
2502 * In usual, we do css_get() when we remember memcg pointer.
2503 * But in this case, we keep res->usage until end of a series of
2504 * uncharges. Then, it's ok to ignore memcg's refcnt.
2505 */
2506 if (!batch->memcg)
2507 batch->memcg = mem;
2508 /*
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002509 * do_batch > 0 when unmapping pages or inode invalidate/truncate.
2510 * In those cases, all pages freed continously can be expected to be in
2511 * the same cgroup and we have chance to coalesce uncharges.
2512 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
2513 * because we want to do uncharge as soon as possible.
2514 */
2515
2516 if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
2517 goto direct_uncharge;
2518
Andrea Arcangeliec168512011-01-13 15:46:56 -08002519 if (page_size != PAGE_SIZE)
2520 goto direct_uncharge;
2521
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002522 /*
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002523 * In typical case, batch->memcg == mem. This means we can
2524 * merge a series of uncharges to an uncharge of res_counter.
2525 * If not, we uncharge res_counter ony by one.
2526 */
2527 if (batch->memcg != mem)
2528 goto direct_uncharge;
2529 /* remember freed charge and uncharge it later */
2530 batch->bytes += PAGE_SIZE;
2531 if (uncharge_memsw)
2532 batch->memsw_bytes += PAGE_SIZE;
2533 return;
2534direct_uncharge:
Andrea Arcangeliec168512011-01-13 15:46:56 -08002535 res_counter_uncharge(&mem->res, page_size);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002536 if (uncharge_memsw)
Andrea Arcangeliec168512011-01-13 15:46:56 -08002537 res_counter_uncharge(&mem->memsw, page_size);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002538 if (unlikely(batch->memcg != mem))
2539 memcg_oom_recover(mem);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002540 return;
2541}
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002542
Balbir Singh8697d332008-02-07 00:13:59 -08002543/*
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002544 * uncharge if !page_mapped(page)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002545 */
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002546static struct mem_cgroup *
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002547__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002548{
Daisuke Nishimura152c9cc2011-01-13 15:46:56 -08002549 int i;
2550 int count;
Hugh Dickins82895462008-03-04 14:29:08 -08002551 struct page_cgroup *pc;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002552 struct mem_cgroup *mem = NULL;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002553 int page_size = PAGE_SIZE;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002554
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002555 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002556 return NULL;
Balbir Singh40779602008-04-04 14:29:59 -07002557
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002558 if (PageSwapCache(page))
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002559 return NULL;
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002560
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08002561 if (PageTransHuge(page)) {
Andrea Arcangeliec168512011-01-13 15:46:56 -08002562 page_size <<= compound_order(page);
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08002563 VM_BUG_ON(!PageTransHuge(page));
2564 }
Andrea Arcangeliec168512011-01-13 15:46:56 -08002565
Daisuke Nishimura152c9cc2011-01-13 15:46:56 -08002566 count = page_size >> PAGE_SHIFT;
Balbir Singh8697d332008-02-07 00:13:59 -08002567 /*
Balbir Singh3c541e12008-02-07 00:14:41 -08002568 * Check if our page_cgroup is valid
Balbir Singh8697d332008-02-07 00:13:59 -08002569 */
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002570 pc = lookup_page_cgroup(page);
2571 if (unlikely(!pc || !PageCgroupUsed(pc)))
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002572 return NULL;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002573
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002574 lock_page_cgroup(pc);
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002575
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002576 mem = pc->mem_cgroup;
2577
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002578 if (!PageCgroupUsed(pc))
2579 goto unlock_out;
2580
2581 switch (ctype) {
2582 case MEM_CGROUP_CHARGE_TYPE_MAPPED:
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07002583 case MEM_CGROUP_CHARGE_TYPE_DROP:
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002584 /* See mem_cgroup_prepare_migration() */
2585 if (page_mapped(page) || PageCgroupMigration(pc))
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002586 goto unlock_out;
2587 break;
2588 case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
2589 if (!PageAnon(page)) { /* Shared memory */
2590 if (page->mapping && !page_is_file_cache(page))
2591 goto unlock_out;
2592 } else if (page_mapped(page)) /* Anon */
2593 goto unlock_out;
2594 break;
2595 default:
2596 break;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002597 }
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002598
Daisuke Nishimura152c9cc2011-01-13 15:46:56 -08002599 for (i = 0; i < count; i++)
2600 mem_cgroup_charge_statistics(mem, pc + i, false);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07002601
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002602 ClearPageCgroupUsed(pc);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08002603 /*
2604 * pc->mem_cgroup is not cleared here. It will be accessed when it's
2605 * freed from LRU. This is safe because uncharged page is expected not
2606 * to be reused (freed soon). Exception is SwapCache, it's handled by
2607 * special functions.
2608 */
Hugh Dickinsb9c565d2008-03-04 14:29:11 -08002609
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002610 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002611 /*
2612 * even after unlock, we have mem->res.usage here and this memcg
2613 * will never be freed.
2614 */
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08002615 memcg_check_events(mem, page);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002616 if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
2617 mem_cgroup_swap_statistics(mem, true);
2618 mem_cgroup_get(mem);
2619 }
2620 if (!mem_cgroup_is_root(mem))
Andrea Arcangeliec168512011-01-13 15:46:56 -08002621 __do_uncharge(mem, ctype, page_size);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08002622
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002623 return mem;
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002624
2625unlock_out:
2626 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002627 return NULL;
Balbir Singh3c541e12008-02-07 00:14:41 -08002628}
2629
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002630void mem_cgroup_uncharge_page(struct page *page)
2631{
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002632 /* early check. */
2633 if (page_mapped(page))
2634 return;
2635 if (page->mapping && !PageAnon(page))
2636 return;
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002637 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
2638}
2639
2640void mem_cgroup_uncharge_cache_page(struct page *page)
2641{
2642 VM_BUG_ON(page_mapped(page));
KAMEZAWA Hiroyukib7abea92008-10-18 20:28:09 -07002643 VM_BUG_ON(page->mapping);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002644 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
2645}
2646
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002647/*
2648 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
2649 * In that cases, pages are freed continuously and we can expect pages
2650 * are in the same memcg. All these calls itself limits the number of
2651 * pages freed at once, then uncharge_start/end() is called properly.
2652 * This may be called prural(2) times in a context,
2653 */
2654
2655void mem_cgroup_uncharge_start(void)
2656{
2657 current->memcg_batch.do_batch++;
2658 /* We can do nest. */
2659 if (current->memcg_batch.do_batch == 1) {
2660 current->memcg_batch.memcg = NULL;
2661 current->memcg_batch.bytes = 0;
2662 current->memcg_batch.memsw_bytes = 0;
2663 }
2664}
2665
2666void mem_cgroup_uncharge_end(void)
2667{
2668 struct memcg_batch_info *batch = &current->memcg_batch;
2669
2670 if (!batch->do_batch)
2671 return;
2672
2673 batch->do_batch--;
2674 if (batch->do_batch) /* If stacked, do nothing. */
2675 return;
2676
2677 if (!batch->memcg)
2678 return;
2679 /*
2680 * This "batch->memcg" is valid without any css_get/put etc...
2681 * bacause we hide charges behind us.
2682 */
2683 if (batch->bytes)
2684 res_counter_uncharge(&batch->memcg->res, batch->bytes);
2685 if (batch->memsw_bytes)
2686 res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002687 memcg_oom_recover(batch->memcg);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08002688 /* forget this pointer (for sanity check) */
2689 batch->memcg = NULL;
2690}
2691
Daisuke Nishimurae767e052009-05-28 14:34:28 -07002692#ifdef CONFIG_SWAP
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002693/*
Daisuke Nishimurae767e052009-05-28 14:34:28 -07002694 * called after __delete_from_swap_cache() and drop "page" account.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002695 * memcg information is recorded to swap_cgroup of "ent"
2696 */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07002697void
2698mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002699{
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002700 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07002701 int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002702
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07002703 if (!swapout) /* this was a swap cache but the swap is unused ! */
2704 ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
2705
2706 memcg = __mem_cgroup_uncharge_common(page, ctype);
2707
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002708 /*
2709 * record memcg information, if swapout && memcg != NULL,
2710 * mem_cgroup_get() was called in uncharge().
2711 */
2712 if (do_swap_account && swapout && memcg)
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002713 swap_cgroup_record(ent, css_id(&memcg->css));
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002714}
Daisuke Nishimurae767e052009-05-28 14:34:28 -07002715#endif
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08002716
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002717#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
2718/*
2719 * called from swap_entry_free(). remove record in swap_cgroup and
2720 * uncharge "memsw" account.
2721 */
2722void mem_cgroup_uncharge_swap(swp_entry_t ent)
2723{
2724 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002725 unsigned short id;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002726
2727 if (!do_swap_account)
2728 return;
2729
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002730 id = swap_cgroup_record(ent, 0);
2731 rcu_read_lock();
2732 memcg = mem_cgroup_lookup(id);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002733 if (memcg) {
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002734 /*
2735 * We uncharge this because swap is freed.
2736 * This memcg can be obsolete one. We avoid calling css_tryget
2737 */
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002738 if (!mem_cgroup_is_root(memcg))
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -07002739 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002740 mem_cgroup_swap_statistics(memcg, false);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002741 mem_cgroup_put(memcg);
2742 }
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002743 rcu_read_unlock();
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002744}
Daisuke Nishimura02491442010-03-10 15:22:17 -08002745
2746/**
2747 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2748 * @entry: swap entry to be moved
2749 * @from: mem_cgroup which the entry is moved from
2750 * @to: mem_cgroup which the entry is moved to
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002751 * @need_fixup: whether we should fixup res_counters and refcounts.
Daisuke Nishimura02491442010-03-10 15:22:17 -08002752 *
2753 * It succeeds only when the swap_cgroup's record for this entry is the same
2754 * as the mem_cgroup's id of @from.
2755 *
2756 * Returns 0 on success, -EINVAL on failure.
2757 *
2758 * The caller must have charged to @to, IOW, called res_counter_charge() about
2759 * both res and memsw, and called css_get().
2760 */
2761static int mem_cgroup_move_swap_account(swp_entry_t entry,
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002762 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
Daisuke Nishimura02491442010-03-10 15:22:17 -08002763{
2764 unsigned short old_id, new_id;
2765
2766 old_id = css_id(&from->css);
2767 new_id = css_id(&to->css);
2768
2769 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08002770 mem_cgroup_swap_statistics(from, false);
Daisuke Nishimura02491442010-03-10 15:22:17 -08002771 mem_cgroup_swap_statistics(to, true);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002772 /*
2773 * This function is only called from task migration context now.
2774 * It postpones res_counter and refcount handling till the end
2775 * of task migration(mem_cgroup_clear_mc()) for performance
2776 * improvement. But we cannot postpone mem_cgroup_get(to)
2777 * because if the process that has been moved to @to does
2778 * swap-in, the refcount of @to might be decreased to 0.
2779 */
Daisuke Nishimura02491442010-03-10 15:22:17 -08002780 mem_cgroup_get(to);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002781 if (need_fixup) {
2782 if (!mem_cgroup_is_root(from))
2783 res_counter_uncharge(&from->memsw, PAGE_SIZE);
2784 mem_cgroup_put(from);
2785 /*
2786 * we charged both to->res and to->memsw, so we should
2787 * uncharge to->res.
2788 */
2789 if (!mem_cgroup_is_root(to))
2790 res_counter_uncharge(&to->res, PAGE_SIZE);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002791 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08002792 return 0;
2793 }
2794 return -EINVAL;
2795}
2796#else
2797static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08002798 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
Daisuke Nishimura02491442010-03-10 15:22:17 -08002799{
2800 return -EINVAL;
2801}
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002802#endif
2803
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08002804/*
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002805 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
2806 * page belongs to.
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08002807 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002808int mem_cgroup_prepare_migration(struct page *page,
2809 struct page *newpage, struct mem_cgroup **ptr)
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08002810{
2811 struct page_cgroup *pc;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002812 struct mem_cgroup *mem = NULL;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002813 enum charge_type ctype;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002814 int ret = 0;
Hugh Dickins8869b8f2008-03-04 14:29:09 -08002815
Andrea Arcangeliec168512011-01-13 15:46:56 -08002816 VM_BUG_ON(PageTransHuge(page));
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002817 if (mem_cgroup_disabled())
Balbir Singh40779602008-04-04 14:29:59 -07002818 return 0;
2819
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002820 pc = lookup_page_cgroup(page);
2821 lock_page_cgroup(pc);
2822 if (PageCgroupUsed(pc)) {
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002823 mem = pc->mem_cgroup;
2824 css_get(&mem->css);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002825 /*
2826 * At migrating an anonymous page, its mapcount goes down
2827 * to 0 and uncharge() will be called. But, even if it's fully
2828 * unmapped, migration may fail and this page has to be
2829 * charged again. We set MIGRATION flag here and delay uncharge
2830 * until end_migration() is called
2831 *
2832 * Corner Case Thinking
2833 * A)
2834 * When the old page was mapped as Anon and it's unmap-and-freed
2835 * while migration was ongoing.
2836 * If unmap finds the old page, uncharge() of it will be delayed
2837 * until end_migration(). If unmap finds a new page, it's
2838 * uncharged when it make mapcount to be 1->0. If unmap code
2839 * finds swap_migration_entry, the new page will not be mapped
2840 * and end_migration() will find it(mapcount==0).
2841 *
2842 * B)
2843 * When the old page was mapped but migraion fails, the kernel
2844 * remaps it. A charge for it is kept by MIGRATION flag even
2845 * if mapcount goes down to 0. We can do remap successfully
2846 * without charging it again.
2847 *
2848 * C)
2849 * The "old" page is under lock_page() until the end of
2850 * migration, so, the old page itself will not be swapped-out.
2851 * If the new page is swapped out before end_migraton, our
2852 * hook to usual swap-out path will catch the event.
2853 */
2854 if (PageAnon(page))
2855 SetPageCgroupMigration(pc);
Hugh Dickinsb9c565d2008-03-04 14:29:11 -08002856 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002857 unlock_page_cgroup(pc);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002858 /*
2859 * If the page is not charged at this point,
2860 * we return here.
2861 */
2862 if (!mem)
2863 return 0;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002864
Andrea Arcangeli93d5c9b2010-04-23 13:17:39 -04002865 *ptr = mem;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002866 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, ptr, false, PAGE_SIZE);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002867 css_put(&mem->css);/* drop extra refcnt */
2868 if (ret || *ptr == NULL) {
2869 if (PageAnon(page)) {
2870 lock_page_cgroup(pc);
2871 ClearPageCgroupMigration(pc);
2872 unlock_page_cgroup(pc);
2873 /*
2874 * The old page may be fully unmapped while we kept it.
2875 */
2876 mem_cgroup_uncharge_page(page);
2877 }
2878 return -ENOMEM;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002879 }
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002880 /*
2881 * We charge new page before it's used/mapped. So, even if unlock_page()
2882 * is called before end_migration, we can catch all events on this new
2883 * page. In the case new page is migrated but not remapped, new page's
2884 * mapcount will be finally 0 and we call uncharge in end_migration().
2885 */
2886 pc = lookup_page_cgroup(newpage);
2887 if (PageAnon(page))
2888 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
2889 else if (page_is_file_cache(page))
2890 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
2891 else
2892 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002893 __mem_cgroup_commit_charge(mem, pc, ctype, PAGE_SIZE);
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002894 return ret;
2895}
Hugh Dickinsfb59e9f2008-03-04 14:29:16 -08002896
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002897/* remove redundant charge if migration failed*/
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002898void mem_cgroup_end_migration(struct mem_cgroup *mem,
Daisuke Nishimura50de1dd2011-01-13 15:47:43 -08002899 struct page *oldpage, struct page *newpage, bool migration_ok)
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002900{
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002901 struct page *used, *unused;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002902 struct page_cgroup *pc;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002903
2904 if (!mem)
2905 return;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002906 /* blocks rmdir() */
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002907 cgroup_exclude_rmdir(&mem->css);
Daisuke Nishimura50de1dd2011-01-13 15:47:43 -08002908 if (!migration_ok) {
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002909 used = oldpage;
2910 unused = newpage;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002911 } else {
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002912 used = newpage;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002913 unused = oldpage;
2914 }
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002915 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002916 * We disallowed uncharge of pages under migration because mapcount
2917 * of the page goes down to zero, temporarly.
2918 * Clear the flag and check the page should be charged.
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002919 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002920 pc = lookup_page_cgroup(oldpage);
2921 lock_page_cgroup(pc);
2922 ClearPageCgroupMigration(pc);
2923 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002924
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002925 __mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE);
2926
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002927 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002928 * If a page is a file cache, radix-tree replacement is very atomic
2929 * and we can skip this check. When it was an Anon page, its mapcount
2930 * goes down to 0. But because we added MIGRATION flage, it's not
2931 * uncharged yet. There are several case but page->mapcount check
2932 * and USED bit check in mem_cgroup_uncharge_page() will do enough
2933 * check. (see prepare_charge() also)
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08002934 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002935 if (PageAnon(used))
2936 mem_cgroup_uncharge_page(used);
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002937 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07002938 * At migration, we may charge account against cgroup which has no
2939 * tasks.
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002940 * So, rmdir()->pre_destroy() can be called while we do this charge.
2941 * In that case, we need to call pre_destroy() again. check it here.
2942 */
2943 cgroup_release_and_wakeup_rmdir(&mem->css);
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08002944}
Pavel Emelianov78fb7462008-02-07 00:13:51 -08002945
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08002946/*
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002947 * A call to try to shrink memory usage on charge failure at shmem's swapin.
2948 * Calling hierarchical_reclaim is not enough because we should update
2949 * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
2950 * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
2951 * not from the memcg which this page would be charged to.
2952 * try_charge_swapin does all of these works properly.
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002953 */
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002954int mem_cgroup_shmem_charge_fallback(struct page *page,
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002955 struct mm_struct *mm,
2956 gfp_t gfp_mask)
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002957{
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002958 struct mem_cgroup *mem = NULL;
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002959 int ret;
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002960
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08002961 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07002962 return 0;
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002963
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002964 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2965 if (!ret)
2966 mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002967
Daisuke Nishimuraae3abae2009-04-30 15:08:19 -07002968 return ret;
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07002969}
2970
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002971static DEFINE_MUTEX(set_limit_mutex);
2972
KOSAKI Motohirod38d2a72009-01-06 14:39:44 -08002973static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002974 unsigned long long val)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002975{
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07002976 int retry_count;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002977 u64 memswlimit, memlimit;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002978 int ret = 0;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07002979 int children = mem_cgroup_count_children(memcg);
2980 u64 curusage, oldusage;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002981 int enlarge;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07002982
2983 /*
2984 * For keeping hierarchical_reclaim simple, how long we should retry
2985 * is depends on callers. We set our retry-count to be function
2986 * of # of children which we should visit in this loop.
2987 */
2988 retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
2989
2990 oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002991
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002992 enlarge = 0;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002993 while (retry_count) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07002994 if (signal_pending(current)) {
2995 ret = -EINTR;
2996 break;
2997 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08002998 /*
2999 * Rather than hide all in some function, I do this in
3000 * open coded manner. You see what this really does.
3001 * We have to guarantee mem->res.limit < mem->memsw.limit.
3002 */
3003 mutex_lock(&set_limit_mutex);
3004 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3005 if (memswlimit < val) {
3006 ret = -EINVAL;
3007 mutex_unlock(&set_limit_mutex);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003008 break;
3009 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003010
3011 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3012 if (memlimit < val)
3013 enlarge = 1;
3014
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003015 ret = res_counter_set_limit(&memcg->res, val);
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07003016 if (!ret) {
3017 if (memswlimit == val)
3018 memcg->memsw_is_minimum = true;
3019 else
3020 memcg->memsw_is_minimum = false;
3021 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003022 mutex_unlock(&set_limit_mutex);
3023
3024 if (!ret)
3025 break;
3026
Bob Liuaa20d482009-12-15 16:47:14 -08003027 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
Balbir Singh4e416952009-09-23 15:56:39 -07003028 MEM_CGROUP_RECLAIM_SHRINK);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003029 curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
3030 /* Usage is reduced ? */
3031 if (curusage >= oldusage)
3032 retry_count--;
3033 else
3034 oldusage = curusage;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003035 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003036 if (!ret && enlarge)
3037 memcg_oom_recover(memcg);
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08003038
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003039 return ret;
3040}
3041
Li Zefan338c8432009-06-17 16:27:15 -07003042static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
3043 unsigned long long val)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003044{
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003045 int retry_count;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003046 u64 memlimit, memswlimit, oldusage, curusage;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003047 int children = mem_cgroup_count_children(memcg);
3048 int ret = -EBUSY;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003049 int enlarge = 0;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003050
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003051 /* see mem_cgroup_resize_res_limit */
3052 retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
3053 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003054 while (retry_count) {
3055 if (signal_pending(current)) {
3056 ret = -EINTR;
3057 break;
3058 }
3059 /*
3060 * Rather than hide all in some function, I do this in
3061 * open coded manner. You see what this really does.
3062 * We have to guarantee mem->res.limit < mem->memsw.limit.
3063 */
3064 mutex_lock(&set_limit_mutex);
3065 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3066 if (memlimit > val) {
3067 ret = -EINVAL;
3068 mutex_unlock(&set_limit_mutex);
3069 break;
3070 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003071 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3072 if (memswlimit < val)
3073 enlarge = 1;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003074 ret = res_counter_set_limit(&memcg->memsw, val);
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07003075 if (!ret) {
3076 if (memlimit == val)
3077 memcg->memsw_is_minimum = true;
3078 else
3079 memcg->memsw_is_minimum = false;
3080 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003081 mutex_unlock(&set_limit_mutex);
3082
3083 if (!ret)
3084 break;
3085
Balbir Singh4e416952009-09-23 15:56:39 -07003086 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
Balbir Singh75822b42009-09-23 15:56:38 -07003087 MEM_CGROUP_RECLAIM_NOSWAP |
3088 MEM_CGROUP_RECLAIM_SHRINK);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003089 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003090 /* Usage is reduced ? */
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003091 if (curusage >= oldusage)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003092 retry_count--;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003093 else
3094 oldusage = curusage;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003095 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003096 if (!ret && enlarge)
3097 memcg_oom_recover(memcg);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003098 return ret;
3099}
3100
Balbir Singh4e416952009-09-23 15:56:39 -07003101unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
KOSAKI Motohiro00918b62010-08-10 18:03:05 -07003102 gfp_t gfp_mask)
Balbir Singh4e416952009-09-23 15:56:39 -07003103{
3104 unsigned long nr_reclaimed = 0;
3105 struct mem_cgroup_per_zone *mz, *next_mz = NULL;
3106 unsigned long reclaimed;
3107 int loop = 0;
3108 struct mem_cgroup_tree_per_zone *mctz;
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07003109 unsigned long long excess;
Balbir Singh4e416952009-09-23 15:56:39 -07003110
3111 if (order > 0)
3112 return 0;
3113
KOSAKI Motohiro00918b62010-08-10 18:03:05 -07003114 mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
Balbir Singh4e416952009-09-23 15:56:39 -07003115 /*
3116 * This loop can run a while, specially if mem_cgroup's continuously
3117 * keep exceeding their soft limit and putting the system under
3118 * pressure
3119 */
3120 do {
3121 if (next_mz)
3122 mz = next_mz;
3123 else
3124 mz = mem_cgroup_largest_soft_limit_node(mctz);
3125 if (!mz)
3126 break;
3127
3128 reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
3129 gfp_mask,
3130 MEM_CGROUP_RECLAIM_SOFT);
3131 nr_reclaimed += reclaimed;
3132 spin_lock(&mctz->lock);
3133
3134 /*
3135 * If we failed to reclaim anything from this memory cgroup
3136 * it is time to move on to the next cgroup
3137 */
3138 next_mz = NULL;
3139 if (!reclaimed) {
3140 do {
3141 /*
3142 * Loop until we find yet another one.
3143 *
3144 * By the time we get the soft_limit lock
3145 * again, someone might have aded the
3146 * group back on the RB tree. Iterate to
3147 * make sure we get a different mem.
3148 * mem_cgroup_largest_soft_limit_node returns
3149 * NULL if no other cgroup is present on
3150 * the tree
3151 */
3152 next_mz =
3153 __mem_cgroup_largest_soft_limit_node(mctz);
3154 if (next_mz == mz) {
3155 css_put(&next_mz->mem->css);
3156 next_mz = NULL;
3157 } else /* next_mz == NULL or other memcg */
3158 break;
3159 } while (1);
3160 }
Balbir Singh4e416952009-09-23 15:56:39 -07003161 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07003162 excess = res_counter_soft_limit_excess(&mz->mem->res);
Balbir Singh4e416952009-09-23 15:56:39 -07003163 /*
3164 * One school of thought says that we should not add
3165 * back the node to the tree if reclaim returns 0.
3166 * But our reclaim could return 0, simply because due
3167 * to priority we are exposing a smaller subset of
3168 * memory to reclaim from. Consider this as a longer
3169 * term TODO.
3170 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07003171 /* If excess == 0, no tree ops */
3172 __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
Balbir Singh4e416952009-09-23 15:56:39 -07003173 spin_unlock(&mctz->lock);
3174 css_put(&mz->mem->css);
3175 loop++;
3176 /*
3177 * Could not reclaim anything and there are no more
3178 * mem cgroups to try or we seem to be looping without
3179 * reclaiming anything.
3180 */
3181 if (!nr_reclaimed &&
3182 (next_mz == NULL ||
3183 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
3184 break;
3185 } while (!nr_reclaimed);
3186 if (next_mz)
3187 css_put(&next_mz->mem->css);
3188 return nr_reclaimed;
3189}
3190
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07003191/*
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003192 * This routine traverse page_cgroup in given list and drop them all.
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003193 * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
3194 */
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003195static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003196 int node, int zid, enum lru_list lru)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003197{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003198 struct zone *zone;
3199 struct mem_cgroup_per_zone *mz;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003200 struct page_cgroup *pc, *busy;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003201 unsigned long flags, loop;
KAMEZAWA Hiroyuki072c56c12008-02-07 00:14:39 -08003202 struct list_head *list;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003203 int ret = 0;
KAMEZAWA Hiroyuki072c56c12008-02-07 00:14:39 -08003204
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003205 zone = &NODE_DATA(node)->node_zones[zid];
3206 mz = mem_cgroup_zoneinfo(mem, node, zid);
Christoph Lameterb69408e2008-10-18 20:26:14 -07003207 list = &mz->lists[lru];
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003208
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003209 loop = MEM_CGROUP_ZSTAT(mz, lru);
3210 /* give some margin against EBUSY etc...*/
3211 loop += 256;
3212 busy = NULL;
3213 while (loop--) {
3214 ret = 0;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003215 spin_lock_irqsave(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003216 if (list_empty(list)) {
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003217 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003218 break;
3219 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003220 pc = list_entry(list->prev, struct page_cgroup, lru);
3221 if (busy == pc) {
3222 list_move(&pc->lru, list);
Thiago Farina648bcc72010-03-05 13:42:04 -08003223 busy = NULL;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003224 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003225 continue;
3226 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003227 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003228
KAMEZAWA Hiroyuki2c26fdd2009-01-07 18:08:10 -08003229 ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003230 if (ret == -ENOMEM)
3231 break;
3232
3233 if (ret == -EBUSY || ret == -EINVAL) {
3234 /* found lock contention or "pc" is obsolete. */
3235 busy = pc;
3236 cond_resched();
3237 } else
3238 busy = NULL;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003239 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003240
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003241 if (!ret && !list_empty(list))
3242 return -EBUSY;
3243 return ret;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003244}
3245
3246/*
3247 * make mem_cgroup's charge to be 0 if there is no task.
3248 * This enables deleting this mem_cgroup.
3249 */
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003250static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003251{
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003252 int ret;
3253 int node, zid, shrink;
3254 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003255 struct cgroup *cgrp = mem->css.cgroup;
Hugh Dickins8869b8f2008-03-04 14:29:09 -08003256
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003257 css_get(&mem->css);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003258
3259 shrink = 0;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003260 /* should free all ? */
3261 if (free_all)
3262 goto try_to_free;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003263move_account:
Daisuke Nishimurafce66472010-01-15 17:01:30 -08003264 do {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003265 ret = -EBUSY;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003266 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003267 goto out;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003268 ret = -EINTR;
3269 if (signal_pending(current))
3270 goto out;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003271 /* This is for making all *used* pages to be on LRU. */
3272 lru_add_drain_all();
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08003273 drain_all_stock_sync();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003274 ret = 0;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07003275 mem_cgroup_start_move(mem);
KAMEZAWA Hiroyuki299b4ea2009-01-29 14:25:17 -08003276 for_each_node_state(node, N_HIGH_MEMORY) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003277 for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
Christoph Lameterb69408e2008-10-18 20:26:14 -07003278 enum lru_list l;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003279 for_each_lru(l) {
3280 ret = mem_cgroup_force_empty_list(mem,
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003281 node, zid, l);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003282 if (ret)
3283 break;
3284 }
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08003285 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003286 if (ret)
3287 break;
3288 }
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07003289 mem_cgroup_end_move(mem);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003290 memcg_oom_recover(mem);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003291 /* it seems parent cgroup doesn't have enough mem */
3292 if (ret == -ENOMEM)
3293 goto try_to_free;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003294 cond_resched();
Daisuke Nishimurafce66472010-01-15 17:01:30 -08003295 /* "ret" should also be checked to ensure all lists are empty. */
3296 } while (mem->res.usage > 0 || ret);
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003297out:
3298 css_put(&mem->css);
3299 return ret;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003300
3301try_to_free:
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003302 /* returns EBUSY if there is a task or if we come here twice. */
3303 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003304 ret = -EBUSY;
3305 goto out;
3306 }
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003307 /* we call try-to-free pages for make this cgroup empty */
3308 lru_add_drain_all();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003309 /* try to free all pages in this cgroup */
3310 shrink = 1;
3311 while (nr_retries && mem->res.usage > 0) {
3312 int progress;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003313
3314 if (signal_pending(current)) {
3315 ret = -EINTR;
3316 goto out;
3317 }
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003318 progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
3319 false, get_swappiness(mem));
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003320 if (!progress) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003321 nr_retries--;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003322 /* maybe some writeback is necessary */
Jens Axboe8aa7e842009-07-09 14:52:32 +02003323 congestion_wait(BLK_RW_ASYNC, HZ/10);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003324 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003325
3326 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003327 lru_add_drain();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003328 /* try move_account...there may be some *locked* pages. */
Daisuke Nishimurafce66472010-01-15 17:01:30 -08003329 goto move_account;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003330}
3331
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003332int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
3333{
3334 return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
3335}
3336
3337
Balbir Singh18f59ea2009-01-07 18:08:07 -08003338static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
3339{
3340 return mem_cgroup_from_cont(cont)->use_hierarchy;
3341}
3342
3343static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
3344 u64 val)
3345{
3346 int retval = 0;
3347 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
3348 struct cgroup *parent = cont->parent;
3349 struct mem_cgroup *parent_mem = NULL;
3350
3351 if (parent)
3352 parent_mem = mem_cgroup_from_cont(parent);
3353
3354 cgroup_lock();
3355 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02003356 * If parent's use_hierarchy is set, we can't make any modifications
Balbir Singh18f59ea2009-01-07 18:08:07 -08003357 * in the child subtrees. If it is unset, then the change can
3358 * occur, provided the current cgroup has no children.
3359 *
3360 * For the root cgroup, parent_mem is NULL, we allow value to be
3361 * set if there are no children.
3362 */
3363 if ((!parent_mem || !parent_mem->use_hierarchy) &&
3364 (val == 1 || val == 0)) {
3365 if (list_empty(&cont->children))
3366 mem->use_hierarchy = val;
3367 else
3368 retval = -EBUSY;
3369 } else
3370 retval = -EINVAL;
3371 cgroup_unlock();
3372
3373 return retval;
3374}
3375
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003376
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003377static u64 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
3378 enum mem_cgroup_stat_index idx)
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003379{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003380 struct mem_cgroup *iter;
3381 s64 val = 0;
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003382
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003383 /* each per cpu's value can be minus.Then, use s64 */
3384 for_each_mem_cgroup_tree(iter, mem)
3385 val += mem_cgroup_read_stat(iter, idx);
3386
3387 if (val < 0) /* race ? */
3388 val = 0;
3389 return val;
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003390}
3391
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003392static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
3393{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003394 u64 val;
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003395
3396 if (!mem_cgroup_is_root(mem)) {
3397 if (!swap)
3398 return res_counter_read_u64(&mem->res, RES_USAGE);
3399 else
3400 return res_counter_read_u64(&mem->memsw, RES_USAGE);
3401 }
3402
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003403 val = mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE);
3404 val += mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003405
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003406 if (swap)
3407 val += mem_cgroup_get_recursive_idx_stat(mem,
3408 MEM_CGROUP_STAT_SWAPOUT);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003409
3410 return val << PAGE_SHIFT;
3411}
3412
Paul Menage2c3daa72008-04-29 00:59:58 -07003413static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003414{
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003415 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003416 u64 val;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003417 int type, name;
3418
3419 type = MEMFILE_TYPE(cft->private);
3420 name = MEMFILE_ATTR(cft->private);
3421 switch (type) {
3422 case _MEM:
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003423 if (name == RES_USAGE)
3424 val = mem_cgroup_usage(mem, false);
3425 else
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003426 val = res_counter_read_u64(&mem->res, name);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003427 break;
3428 case _MEMSWAP:
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003429 if (name == RES_USAGE)
3430 val = mem_cgroup_usage(mem, true);
3431 else
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003432 val = res_counter_read_u64(&mem->memsw, name);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003433 break;
3434 default:
3435 BUG();
3436 break;
3437 }
3438 return val;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003439}
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003440/*
3441 * The user of this function is...
3442 * RES_LIMIT.
3443 */
Paul Menage856c13a2008-07-25 01:47:04 -07003444static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
3445 const char *buffer)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003446{
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003447 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003448 int type, name;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003449 unsigned long long val;
3450 int ret;
3451
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003452 type = MEMFILE_TYPE(cft->private);
3453 name = MEMFILE_ATTR(cft->private);
3454 switch (name) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003455 case RES_LIMIT:
Balbir Singh4b3bde42009-09-23 15:56:32 -07003456 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3457 ret = -EINVAL;
3458 break;
3459 }
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003460 /* This function does all necessary parse...reuse it */
3461 ret = res_counter_memparse_write_strategy(buffer, &val);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003462 if (ret)
3463 break;
3464 if (type == _MEM)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003465 ret = mem_cgroup_resize_limit(memcg, val);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003466 else
3467 ret = mem_cgroup_resize_memsw_limit(memcg, val);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003468 break;
Balbir Singh296c81d2009-09-23 15:56:36 -07003469 case RES_SOFT_LIMIT:
3470 ret = res_counter_memparse_write_strategy(buffer, &val);
3471 if (ret)
3472 break;
3473 /*
3474 * For memsw, soft limits are hard to implement in terms
3475 * of semantics, for now, we support soft limits for
3476 * control without swap
3477 */
3478 if (type == _MEM)
3479 ret = res_counter_set_soft_limit(&memcg->res, val);
3480 else
3481 ret = -EINVAL;
3482 break;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003483 default:
3484 ret = -EINVAL; /* should be BUG() ? */
3485 break;
3486 }
3487 return ret;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003488}
3489
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08003490static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
3491 unsigned long long *mem_limit, unsigned long long *memsw_limit)
3492{
3493 struct cgroup *cgroup;
3494 unsigned long long min_limit, min_memsw_limit, tmp;
3495
3496 min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3497 min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3498 cgroup = memcg->css.cgroup;
3499 if (!memcg->use_hierarchy)
3500 goto out;
3501
3502 while (cgroup->parent) {
3503 cgroup = cgroup->parent;
3504 memcg = mem_cgroup_from_cont(cgroup);
3505 if (!memcg->use_hierarchy)
3506 break;
3507 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
3508 min_limit = min(min_limit, tmp);
3509 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3510 min_memsw_limit = min(min_memsw_limit, tmp);
3511 }
3512out:
3513 *mem_limit = min_limit;
3514 *memsw_limit = min_memsw_limit;
3515 return;
3516}
3517
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003518static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003519{
3520 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003521 int type, name;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003522
3523 mem = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003524 type = MEMFILE_TYPE(event);
3525 name = MEMFILE_ATTR(event);
3526 switch (name) {
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003527 case RES_MAX_USAGE:
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003528 if (type == _MEM)
3529 res_counter_reset_max(&mem->res);
3530 else
3531 res_counter_reset_max(&mem->memsw);
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003532 break;
3533 case RES_FAILCNT:
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003534 if (type == _MEM)
3535 res_counter_reset_failcnt(&mem->res);
3536 else
3537 res_counter_reset_failcnt(&mem->memsw);
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07003538 break;
3539 }
Balbir Singhf64c3f52009-09-23 15:56:37 -07003540
Pavel Emelyanov85cc59d2008-04-29 01:00:20 -07003541 return 0;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07003542}
3543
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003544static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
3545 struct cftype *cft)
3546{
3547 return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
3548}
3549
Daisuke Nishimura02491442010-03-10 15:22:17 -08003550#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003551static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
3552 struct cftype *cft, u64 val)
3553{
3554 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3555
3556 if (val >= (1 << NR_MOVE_TYPE))
3557 return -EINVAL;
3558 /*
3559 * We check this value several times in both in can_attach() and
3560 * attach(), so we need cgroup lock to prevent this value from being
3561 * inconsistent.
3562 */
3563 cgroup_lock();
3564 mem->move_charge_at_immigrate = val;
3565 cgroup_unlock();
3566
3567 return 0;
3568}
Daisuke Nishimura02491442010-03-10 15:22:17 -08003569#else
3570static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
3571 struct cftype *cft, u64 val)
3572{
3573 return -ENOSYS;
3574}
3575#endif
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08003576
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003577
3578/* For read statistics */
3579enum {
3580 MCS_CACHE,
3581 MCS_RSS,
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -08003582 MCS_FILE_MAPPED,
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003583 MCS_PGPGIN,
3584 MCS_PGPGOUT,
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003585 MCS_SWAP,
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003586 MCS_INACTIVE_ANON,
3587 MCS_ACTIVE_ANON,
3588 MCS_INACTIVE_FILE,
3589 MCS_ACTIVE_FILE,
3590 MCS_UNEVICTABLE,
3591 NR_MCS_STAT,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003592};
3593
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003594struct mcs_total_stat {
3595 s64 stat[NR_MCS_STAT];
3596};
3597
3598struct {
3599 char *local_name;
3600 char *total_name;
3601} memcg_stat_strings[NR_MCS_STAT] = {
3602 {"cache", "total_cache"},
3603 {"rss", "total_rss"},
Balbir Singhd69b0422009-06-17 16:26:34 -07003604 {"mapped_file", "total_mapped_file"},
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003605 {"pgpgin", "total_pgpgin"},
3606 {"pgpgout", "total_pgpgout"},
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003607 {"swap", "total_swap"},
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003608 {"inactive_anon", "total_inactive_anon"},
3609 {"active_anon", "total_active_anon"},
3610 {"inactive_file", "total_inactive_file"},
3611 {"active_file", "total_active_file"},
3612 {"unevictable", "total_unevictable"}
3613};
3614
3615
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003616static void
3617mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003618{
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003619 s64 val;
3620
3621 /* per cpu stat */
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003622 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003623 s->stat[MCS_CACHE] += val * PAGE_SIZE;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003624 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003625 s->stat[MCS_RSS] += val * PAGE_SIZE;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003626 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED);
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -08003627 s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003628 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003629 s->stat[MCS_PGPGIN] += val;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003630 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003631 s->stat[MCS_PGPGOUT] += val;
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003632 if (do_swap_account) {
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003633 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003634 s->stat[MCS_SWAP] += val * PAGE_SIZE;
3635 }
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003636
3637 /* per zone stat */
3638 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
3639 s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
3640 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
3641 s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
3642 val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
3643 s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
3644 val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
3645 s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
3646 val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
3647 s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003648}
3649
3650static void
3651mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
3652{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003653 struct mem_cgroup *iter;
3654
3655 for_each_mem_cgroup_tree(iter, mem)
3656 mem_cgroup_get_local_stat(iter, s);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003657}
3658
Paul Menagec64745c2008-04-29 01:00:02 -07003659static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
3660 struct cgroup_map_cb *cb)
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003661{
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003662 struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003663 struct mcs_total_stat mystat;
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003664 int i;
3665
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003666 memset(&mystat, 0, sizeof(mystat));
3667 mem_cgroup_get_local_stat(mem_cont, &mystat);
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003668
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003669 for (i = 0; i < NR_MCS_STAT; i++) {
3670 if (i == MCS_SWAP && !do_swap_account)
3671 continue;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003672 cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003673 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08003674
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003675 /* Hierarchical information */
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08003676 {
3677 unsigned long long limit, memsw_limit;
3678 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
3679 cb->fill(cb, "hierarchical_memory_limit", limit);
3680 if (do_swap_account)
3681 cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
3682 }
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003683
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003684 memset(&mystat, 0, sizeof(mystat));
3685 mem_cgroup_get_total_stat(mem_cont, &mystat);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003686 for (i = 0; i < NR_MCS_STAT; i++) {
3687 if (i == MCS_SWAP && !do_swap_account)
3688 continue;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003689 cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07003690 }
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07003691
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003692#ifdef CONFIG_DEBUG_VM
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08003693 cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08003694
3695 {
3696 int nid, zid;
3697 struct mem_cgroup_per_zone *mz;
3698 unsigned long recent_rotated[2] = {0, 0};
3699 unsigned long recent_scanned[2] = {0, 0};
3700
3701 for_each_online_node(nid)
3702 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
3703 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
3704
3705 recent_rotated[0] +=
3706 mz->reclaim_stat.recent_rotated[0];
3707 recent_rotated[1] +=
3708 mz->reclaim_stat.recent_rotated[1];
3709 recent_scanned[0] +=
3710 mz->reclaim_stat.recent_scanned[0];
3711 recent_scanned[1] +=
3712 mz->reclaim_stat.recent_scanned[1];
3713 }
3714 cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
3715 cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
3716 cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
3717 cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
3718 }
3719#endif
3720
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08003721 return 0;
3722}
3723
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003724static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
3725{
3726 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3727
3728 return get_swappiness(memcg);
3729}
3730
3731static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
3732 u64 val)
3733{
3734 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3735 struct mem_cgroup *parent;
Li Zefan068b38c2009-01-15 13:51:26 -08003736
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003737 if (val > 100)
3738 return -EINVAL;
3739
3740 if (cgrp->parent == NULL)
3741 return -EINVAL;
3742
3743 parent = mem_cgroup_from_cont(cgrp->parent);
Li Zefan068b38c2009-01-15 13:51:26 -08003744
3745 cgroup_lock();
3746
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003747 /* If under hierarchy, only empty-root can set this value */
3748 if ((parent->use_hierarchy) ||
Li Zefan068b38c2009-01-15 13:51:26 -08003749 (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
3750 cgroup_unlock();
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003751 return -EINVAL;
Li Zefan068b38c2009-01-15 13:51:26 -08003752 }
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003753
3754 spin_lock(&memcg->reclaim_param_lock);
3755 memcg->swappiness = val;
3756 spin_unlock(&memcg->reclaim_param_lock);
3757
Li Zefan068b38c2009-01-15 13:51:26 -08003758 cgroup_unlock();
3759
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003760 return 0;
3761}
3762
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003763static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3764{
3765 struct mem_cgroup_threshold_ary *t;
3766 u64 usage;
3767 int i;
3768
3769 rcu_read_lock();
3770 if (!swap)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003771 t = rcu_dereference(memcg->thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003772 else
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003773 t = rcu_dereference(memcg->memsw_thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003774
3775 if (!t)
3776 goto unlock;
3777
3778 usage = mem_cgroup_usage(memcg, swap);
3779
3780 /*
3781 * current_threshold points to threshold just below usage.
3782 * If it's not true, a threshold was crossed after last
3783 * call of __mem_cgroup_threshold().
3784 */
Phil Carmody5407a562010-05-26 14:42:42 -07003785 i = t->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003786
3787 /*
3788 * Iterate backward over array of thresholds starting from
3789 * current_threshold and check if a threshold is crossed.
3790 * If none of thresholds below usage is crossed, we read
3791 * only one element of the array here.
3792 */
3793 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3794 eventfd_signal(t->entries[i].eventfd, 1);
3795
3796 /* i = current_threshold + 1 */
3797 i++;
3798
3799 /*
3800 * Iterate forward over array of thresholds starting from
3801 * current_threshold+1 and check if a threshold is crossed.
3802 * If none of thresholds above usage is crossed, we read
3803 * only one element of the array here.
3804 */
3805 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3806 eventfd_signal(t->entries[i].eventfd, 1);
3807
3808 /* Update current_threshold */
Phil Carmody5407a562010-05-26 14:42:42 -07003809 t->current_threshold = i - 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003810unlock:
3811 rcu_read_unlock();
3812}
3813
3814static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3815{
Kirill A. Shutemovad4ca5f2010-10-07 12:59:27 -07003816 while (memcg) {
3817 __mem_cgroup_threshold(memcg, false);
3818 if (do_swap_account)
3819 __mem_cgroup_threshold(memcg, true);
3820
3821 memcg = parent_mem_cgroup(memcg);
3822 }
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003823}
3824
3825static int compare_thresholds(const void *a, const void *b)
3826{
3827 const struct mem_cgroup_threshold *_a = a;
3828 const struct mem_cgroup_threshold *_b = b;
3829
3830 return _a->threshold - _b->threshold;
3831}
3832
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003833static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003834{
3835 struct mem_cgroup_eventfd_list *ev;
3836
3837 list_for_each_entry(ev, &mem->oom_notify, list)
3838 eventfd_signal(ev->eventfd, 1);
3839 return 0;
3840}
3841
3842static void mem_cgroup_oom_notify(struct mem_cgroup *mem)
3843{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003844 struct mem_cgroup *iter;
3845
3846 for_each_mem_cgroup_tree(iter, mem)
3847 mem_cgroup_oom_notify_cb(iter);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003848}
3849
3850static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
3851 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003852{
3853 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003854 struct mem_cgroup_thresholds *thresholds;
3855 struct mem_cgroup_threshold_ary *new;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003856 int type = MEMFILE_TYPE(cft->private);
3857 u64 threshold, usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003858 int i, size, ret;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003859
3860 ret = res_counter_memparse_write_strategy(args, &threshold);
3861 if (ret)
3862 return ret;
3863
3864 mutex_lock(&memcg->thresholds_lock);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003865
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003866 if (type == _MEM)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003867 thresholds = &memcg->thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003868 else if (type == _MEMSWAP)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003869 thresholds = &memcg->memsw_thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003870 else
3871 BUG();
3872
3873 usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
3874
3875 /* Check if a threshold crossed before adding a new one */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003876 if (thresholds->primary)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003877 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3878
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003879 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003880
3881 /* Allocate memory for new array of thresholds */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003882 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003883 GFP_KERNEL);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003884 if (!new) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003885 ret = -ENOMEM;
3886 goto unlock;
3887 }
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003888 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003889
3890 /* Copy thresholds (if any) to new array */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003891 if (thresholds->primary) {
3892 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003893 sizeof(struct mem_cgroup_threshold));
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003894 }
3895
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003896 /* Add new threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003897 new->entries[size - 1].eventfd = eventfd;
3898 new->entries[size - 1].threshold = threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003899
3900 /* Sort thresholds. Registering of new threshold isn't time-critical */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003901 sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003902 compare_thresholds, NULL);
3903
3904 /* Find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003905 new->current_threshold = -1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003906 for (i = 0; i < size; i++) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003907 if (new->entries[i].threshold < usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003908 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003909 * new->current_threshold will not be used until
3910 * rcu_assign_pointer(), so it's safe to increment
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003911 * it here.
3912 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003913 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003914 }
3915 }
3916
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003917 /* Free old spare buffer and save old primary buffer as spare */
3918 kfree(thresholds->spare);
3919 thresholds->spare = thresholds->primary;
3920
3921 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003922
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003923 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003924 synchronize_rcu();
3925
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003926unlock:
3927 mutex_unlock(&memcg->thresholds_lock);
3928
3929 return ret;
3930}
3931
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003932static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07003933 struct cftype *cft, struct eventfd_ctx *eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003934{
3935 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003936 struct mem_cgroup_thresholds *thresholds;
3937 struct mem_cgroup_threshold_ary *new;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003938 int type = MEMFILE_TYPE(cft->private);
3939 u64 usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003940 int i, j, size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003941
3942 mutex_lock(&memcg->thresholds_lock);
3943 if (type == _MEM)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003944 thresholds = &memcg->thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003945 else if (type == _MEMSWAP)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003946 thresholds = &memcg->memsw_thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003947 else
3948 BUG();
3949
3950 /*
3951 * Something went wrong if we trying to unregister a threshold
3952 * if we don't have thresholds
3953 */
3954 BUG_ON(!thresholds);
3955
3956 usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
3957
3958 /* Check if a threshold crossed before removing */
3959 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3960
3961 /* Calculate new number of threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003962 size = 0;
3963 for (i = 0; i < thresholds->primary->size; i++) {
3964 if (thresholds->primary->entries[i].eventfd != eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003965 size++;
3966 }
3967
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003968 new = thresholds->spare;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003969
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003970 /* Set thresholds array to NULL if we don't have thresholds */
3971 if (!size) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003972 kfree(new);
3973 new = NULL;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003974 goto swap_buffers;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003975 }
3976
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003977 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003978
3979 /* Copy thresholds and find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003980 new->current_threshold = -1;
3981 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3982 if (thresholds->primary->entries[i].eventfd == eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003983 continue;
3984
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003985 new->entries[j] = thresholds->primary->entries[i];
3986 if (new->entries[j].threshold < usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003987 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003988 * new->current_threshold will not be used
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003989 * until rcu_assign_pointer(), so it's safe to increment
3990 * it here.
3991 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003992 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08003993 }
3994 j++;
3995 }
3996
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07003997swap_buffers:
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07003998 /* Swap primary and spare array */
3999 thresholds->spare = thresholds->primary;
4000 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004001
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004002 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004003 synchronize_rcu();
4004
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004005 mutex_unlock(&memcg->thresholds_lock);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004006}
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004007
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004008static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
4009 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
4010{
4011 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4012 struct mem_cgroup_eventfd_list *event;
4013 int type = MEMFILE_TYPE(cft->private);
4014
4015 BUG_ON(type != _OOM_TYPE);
4016 event = kmalloc(sizeof(*event), GFP_KERNEL);
4017 if (!event)
4018 return -ENOMEM;
4019
4020 mutex_lock(&memcg_oom_mutex);
4021
4022 event->eventfd = eventfd;
4023 list_add(&event->list, &memcg->oom_notify);
4024
4025 /* already in OOM ? */
4026 if (atomic_read(&memcg->oom_lock))
4027 eventfd_signal(eventfd, 1);
4028 mutex_unlock(&memcg_oom_mutex);
4029
4030 return 0;
4031}
4032
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004033static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004034 struct cftype *cft, struct eventfd_ctx *eventfd)
4035{
4036 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4037 struct mem_cgroup_eventfd_list *ev, *tmp;
4038 int type = MEMFILE_TYPE(cft->private);
4039
4040 BUG_ON(type != _OOM_TYPE);
4041
4042 mutex_lock(&memcg_oom_mutex);
4043
4044 list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) {
4045 if (ev->eventfd == eventfd) {
4046 list_del(&ev->list);
4047 kfree(ev);
4048 }
4049 }
4050
4051 mutex_unlock(&memcg_oom_mutex);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004052}
4053
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004054static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
4055 struct cftype *cft, struct cgroup_map_cb *cb)
4056{
4057 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4058
4059 cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable);
4060
4061 if (atomic_read(&mem->oom_lock))
4062 cb->fill(cb, "under_oom", 1);
4063 else
4064 cb->fill(cb, "under_oom", 0);
4065 return 0;
4066}
4067
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004068static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
4069 struct cftype *cft, u64 val)
4070{
4071 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4072 struct mem_cgroup *parent;
4073
4074 /* cannot set to root cgroup and only 0 and 1 are allowed */
4075 if (!cgrp->parent || !((val == 0) || (val == 1)))
4076 return -EINVAL;
4077
4078 parent = mem_cgroup_from_cont(cgrp->parent);
4079
4080 cgroup_lock();
4081 /* oom-kill-disable is a flag for subhierarchy. */
4082 if ((parent->use_hierarchy) ||
4083 (mem->use_hierarchy && !list_empty(&cgrp->children))) {
4084 cgroup_unlock();
4085 return -EINVAL;
4086 }
4087 mem->oom_kill_disable = val;
KAMEZAWA Hiroyuki4d845eb2010-06-29 15:05:18 -07004088 if (!val)
4089 memcg_oom_recover(mem);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004090 cgroup_unlock();
4091 return 0;
4092}
4093
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004094static struct cftype mem_cgroup_files[] = {
4095 {
Balbir Singh0eea1032008-02-07 00:13:57 -08004096 .name = "usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004097 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
Paul Menage2c3daa72008-04-29 00:59:58 -07004098 .read_u64 = mem_cgroup_read,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004099 .register_event = mem_cgroup_usage_register_event,
4100 .unregister_event = mem_cgroup_usage_unregister_event,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004101 },
4102 {
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004103 .name = "max_usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004104 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07004105 .trigger = mem_cgroup_reset,
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004106 .read_u64 = mem_cgroup_read,
4107 },
4108 {
Balbir Singh0eea1032008-02-07 00:13:57 -08004109 .name = "limit_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004110 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
Paul Menage856c13a2008-07-25 01:47:04 -07004111 .write_string = mem_cgroup_write,
Paul Menage2c3daa72008-04-29 00:59:58 -07004112 .read_u64 = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004113 },
4114 {
Balbir Singh296c81d2009-09-23 15:56:36 -07004115 .name = "soft_limit_in_bytes",
4116 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4117 .write_string = mem_cgroup_write,
4118 .read_u64 = mem_cgroup_read,
4119 },
4120 {
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004121 .name = "failcnt",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004122 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07004123 .trigger = mem_cgroup_reset,
Paul Menage2c3daa72008-04-29 00:59:58 -07004124 .read_u64 = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004125 },
Balbir Singh8697d332008-02-07 00:13:59 -08004126 {
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004127 .name = "stat",
Paul Menagec64745c2008-04-29 01:00:02 -07004128 .read_map = mem_control_stat_show,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004129 },
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004130 {
4131 .name = "force_empty",
4132 .trigger = mem_cgroup_force_empty_write,
4133 },
Balbir Singh18f59ea2009-01-07 18:08:07 -08004134 {
4135 .name = "use_hierarchy",
4136 .write_u64 = mem_cgroup_hierarchy_write,
4137 .read_u64 = mem_cgroup_hierarchy_read,
4138 },
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004139 {
4140 .name = "swappiness",
4141 .read_u64 = mem_cgroup_swappiness_read,
4142 .write_u64 = mem_cgroup_swappiness_write,
4143 },
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004144 {
4145 .name = "move_charge_at_immigrate",
4146 .read_u64 = mem_cgroup_move_charge_read,
4147 .write_u64 = mem_cgroup_move_charge_write,
4148 },
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004149 {
4150 .name = "oom_control",
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004151 .read_map = mem_cgroup_oom_control_read,
4152 .write_u64 = mem_cgroup_oom_control_write,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004153 .register_event = mem_cgroup_oom_register_event,
4154 .unregister_event = mem_cgroup_oom_unregister_event,
4155 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4156 },
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004157};
4158
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004159#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4160static struct cftype memsw_cgroup_files[] = {
4161 {
4162 .name = "memsw.usage_in_bytes",
4163 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
4164 .read_u64 = mem_cgroup_read,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004165 .register_event = mem_cgroup_usage_register_event,
4166 .unregister_event = mem_cgroup_usage_unregister_event,
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004167 },
4168 {
4169 .name = "memsw.max_usage_in_bytes",
4170 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
4171 .trigger = mem_cgroup_reset,
4172 .read_u64 = mem_cgroup_read,
4173 },
4174 {
4175 .name = "memsw.limit_in_bytes",
4176 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
4177 .write_string = mem_cgroup_write,
4178 .read_u64 = mem_cgroup_read,
4179 },
4180 {
4181 .name = "memsw.failcnt",
4182 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
4183 .trigger = mem_cgroup_reset,
4184 .read_u64 = mem_cgroup_read,
4185 },
4186};
4187
4188static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
4189{
4190 if (!do_swap_account)
4191 return 0;
4192 return cgroup_add_files(cont, ss, memsw_cgroup_files,
4193 ARRAY_SIZE(memsw_cgroup_files));
4194};
4195#else
4196static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
4197{
4198 return 0;
4199}
4200#endif
4201
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004202static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
4203{
4204 struct mem_cgroup_per_node *pn;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004205 struct mem_cgroup_per_zone *mz;
Christoph Lameterb69408e2008-10-18 20:26:14 -07004206 enum lru_list l;
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07004207 int zone, tmp = node;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004208 /*
4209 * This routine is called against possible nodes.
4210 * But it's BUG to call kmalloc() against offline node.
4211 *
4212 * TODO: this routine can waste much memory for nodes which will
4213 * never be onlined. It's better to use memory hotplug callback
4214 * function.
4215 */
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07004216 if (!node_state(node, N_NORMAL_MEMORY))
4217 tmp = -1;
Jesper Juhl17295c82011-01-13 15:47:42 -08004218 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004219 if (!pn)
4220 return 1;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004221
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004222 mem->info.nodeinfo[node] = pn;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004223 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4224 mz = &pn->zoneinfo[zone];
Christoph Lameterb69408e2008-10-18 20:26:14 -07004225 for_each_lru(l)
4226 INIT_LIST_HEAD(&mz->lists[l]);
Balbir Singhf64c3f52009-09-23 15:56:37 -07004227 mz->usage_in_excess = 0;
Balbir Singh4e416952009-09-23 15:56:39 -07004228 mz->on_tree = false;
4229 mz->mem = mem;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004230 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004231 return 0;
4232}
4233
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004234static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
4235{
4236 kfree(mem->info.nodeinfo[node]);
4237}
4238
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004239static struct mem_cgroup *mem_cgroup_alloc(void)
4240{
4241 struct mem_cgroup *mem;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004242 int size = sizeof(struct mem_cgroup);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004243
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004244 /* Can be very big if MAX_NUMNODES is very big */
Jan Blunckc8dad2b2009-01-07 18:07:53 -08004245 if (size < PAGE_SIZE)
Jesper Juhl17295c82011-01-13 15:47:42 -08004246 mem = kzalloc(size, GFP_KERNEL);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004247 else
Jesper Juhl17295c82011-01-13 15:47:42 -08004248 mem = vzalloc(size);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004249
Dan Carpentere7bbcdf2010-03-23 13:35:12 -07004250 if (!mem)
4251 return NULL;
4252
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004253 mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
Dan Carpenterd2e61b82010-11-11 14:05:12 -08004254 if (!mem->stat)
4255 goto out_free;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07004256 spin_lock_init(&mem->pcp_counter_lock);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004257 return mem;
Dan Carpenterd2e61b82010-11-11 14:05:12 -08004258
4259out_free:
4260 if (size < PAGE_SIZE)
4261 kfree(mem);
4262 else
4263 vfree(mem);
4264 return NULL;
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004265}
4266
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004267/*
4268 * At destroying mem_cgroup, references from swap_cgroup can remain.
4269 * (scanning all at force_empty is too costly...)
4270 *
4271 * Instead of clearing all references at force_empty, we remember
4272 * the number of reference from swap_cgroup and free mem_cgroup when
4273 * it goes down to 0.
4274 *
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004275 * Removal of cgroup itself succeeds regardless of refs from swap.
4276 */
4277
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004278static void __mem_cgroup_free(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004279{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004280 int node;
4281
Balbir Singhf64c3f52009-09-23 15:56:37 -07004282 mem_cgroup_remove_from_trees(mem);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004283 free_css_id(&mem_cgroup_subsys, &mem->css);
4284
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004285 for_each_node_state(node, N_POSSIBLE)
4286 free_mem_cgroup_per_zone_info(mem, node);
4287
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004288 free_percpu(mem->stat);
4289 if (sizeof(struct mem_cgroup) < PAGE_SIZE)
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004290 kfree(mem);
4291 else
4292 vfree(mem);
4293}
4294
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004295static void mem_cgroup_get(struct mem_cgroup *mem)
4296{
4297 atomic_inc(&mem->refcnt);
4298}
4299
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004300static void __mem_cgroup_put(struct mem_cgroup *mem, int count)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004301{
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004302 if (atomic_sub_and_test(count, &mem->refcnt)) {
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004303 struct mem_cgroup *parent = parent_mem_cgroup(mem);
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004304 __mem_cgroup_free(mem);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004305 if (parent)
4306 mem_cgroup_put(parent);
4307 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004308}
4309
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004310static void mem_cgroup_put(struct mem_cgroup *mem)
4311{
4312 __mem_cgroup_put(mem, 1);
4313}
4314
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004315/*
4316 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
4317 */
4318static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
4319{
4320 if (!mem->res.parent)
4321 return NULL;
4322 return mem_cgroup_from_res_counter(mem->res.parent, res);
4323}
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004324
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004325#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4326static void __init enable_swap_cgroup(void)
4327{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08004328 if (!mem_cgroup_disabled() && really_do_swap_account)
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004329 do_swap_account = 1;
4330}
4331#else
4332static void __init enable_swap_cgroup(void)
4333{
4334}
4335#endif
4336
Balbir Singhf64c3f52009-09-23 15:56:37 -07004337static int mem_cgroup_soft_limit_tree_init(void)
4338{
4339 struct mem_cgroup_tree_per_node *rtpn;
4340 struct mem_cgroup_tree_per_zone *rtpz;
4341 int tmp, node, zone;
4342
4343 for_each_node_state(node, N_POSSIBLE) {
4344 tmp = node;
4345 if (!node_state(node, N_NORMAL_MEMORY))
4346 tmp = -1;
4347 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
4348 if (!rtpn)
4349 return 1;
4350
4351 soft_limit_tree.rb_tree_per_node[node] = rtpn;
4352
4353 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4354 rtpz = &rtpn->rb_tree_per_zone[zone];
4355 rtpz->rb_root = RB_ROOT;
4356 spin_lock_init(&rtpz->lock);
4357 }
4358 }
4359 return 0;
4360}
4361
Li Zefan0eb253e2009-01-15 13:51:25 -08004362static struct cgroup_subsys_state * __ref
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004363mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
4364{
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004365 struct mem_cgroup *mem, *parent;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004366 long error = -ENOMEM;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004367 int node;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004368
Jan Blunckc8dad2b2009-01-07 18:07:53 -08004369 mem = mem_cgroup_alloc();
4370 if (!mem)
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004371 return ERR_PTR(error);
Pavel Emelianov78fb7462008-02-07 00:13:51 -08004372
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004373 for_each_node_state(node, N_POSSIBLE)
4374 if (alloc_mem_cgroup_per_zone_info(mem, node))
4375 goto free_out;
Balbir Singhf64c3f52009-09-23 15:56:37 -07004376
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004377 /* root ? */
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004378 if (cont->parent == NULL) {
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08004379 int cpu;
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004380 enable_swap_cgroup();
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004381 parent = NULL;
Balbir Singh4b3bde42009-09-23 15:56:32 -07004382 root_mem_cgroup = mem;
Balbir Singhf64c3f52009-09-23 15:56:37 -07004383 if (mem_cgroup_soft_limit_tree_init())
4384 goto free_out;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08004385 for_each_possible_cpu(cpu) {
4386 struct memcg_stock_pcp *stock =
4387 &per_cpu(memcg_stock, cpu);
4388 INIT_WORK(&stock->work, drain_local_stock);
4389 }
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07004390 hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
Balbir Singh18f59ea2009-01-07 18:08:07 -08004391 } else {
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004392 parent = mem_cgroup_from_cont(cont->parent);
Balbir Singh18f59ea2009-01-07 18:08:07 -08004393 mem->use_hierarchy = parent->use_hierarchy;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004394 mem->oom_kill_disable = parent->oom_kill_disable;
Balbir Singh18f59ea2009-01-07 18:08:07 -08004395 }
Balbir Singh28dbc4b2009-01-07 18:08:05 -08004396
Balbir Singh18f59ea2009-01-07 18:08:07 -08004397 if (parent && parent->use_hierarchy) {
4398 res_counter_init(&mem->res, &parent->res);
4399 res_counter_init(&mem->memsw, &parent->memsw);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004400 /*
4401 * We increment refcnt of the parent to ensure that we can
4402 * safely access it on res_counter_charge/uncharge.
4403 * This refcnt will be decremented when freeing this
4404 * mem_cgroup(see mem_cgroup_put).
4405 */
4406 mem_cgroup_get(parent);
Balbir Singh18f59ea2009-01-07 18:08:07 -08004407 } else {
4408 res_counter_init(&mem->res, NULL);
4409 res_counter_init(&mem->memsw, NULL);
4410 }
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004411 mem->last_scanned_child = 0;
KOSAKI Motohiro2733c062009-01-07 18:08:23 -08004412 spin_lock_init(&mem->reclaim_param_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004413 INIT_LIST_HEAD(&mem->oom_notify);
Balbir Singh6d61ef42009-01-07 18:08:06 -08004414
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004415 if (parent)
4416 mem->swappiness = get_swappiness(parent);
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004417 atomic_set(&mem->refcnt, 1);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004418 mem->move_charge_at_immigrate = 0;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004419 mutex_init(&mem->thresholds_lock);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004420 return &mem->css;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004421free_out:
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004422 __mem_cgroup_free(mem);
Balbir Singh4b3bde42009-09-23 15:56:32 -07004423 root_mem_cgroup = NULL;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004424 return ERR_PTR(error);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004425}
4426
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07004427static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08004428 struct cgroup *cont)
4429{
4430 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07004431
4432 return mem_cgroup_force_empty(mem, false);
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08004433}
4434
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004435static void mem_cgroup_destroy(struct cgroup_subsys *ss,
4436 struct cgroup *cont)
4437{
Daisuke Nishimurac268e992009-01-15 13:51:13 -08004438 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
Daisuke Nishimurac268e992009-01-15 13:51:13 -08004439
Daisuke Nishimurac268e992009-01-15 13:51:13 -08004440 mem_cgroup_put(mem);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004441}
4442
4443static int mem_cgroup_populate(struct cgroup_subsys *ss,
4444 struct cgroup *cont)
4445{
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004446 int ret;
4447
4448 ret = cgroup_add_files(cont, ss, mem_cgroup_files,
4449 ARRAY_SIZE(mem_cgroup_files));
4450
4451 if (!ret)
4452 ret = register_memsw_files(cont, ss);
4453 return ret;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004454}
4455
Daisuke Nishimura02491442010-03-10 15:22:17 -08004456#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004457/* Handlers for move charge at task migration. */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004458#define PRECHARGE_COUNT_AT_ONCE 256
4459static int mem_cgroup_do_precharge(unsigned long count)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004460{
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004461 int ret = 0;
4462 int batch_count = PRECHARGE_COUNT_AT_ONCE;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004463 struct mem_cgroup *mem = mc.to;
4464
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004465 if (mem_cgroup_is_root(mem)) {
4466 mc.precharge += count;
4467 /* we don't need css_get for root */
4468 return ret;
4469 }
4470 /* try to charge at once */
4471 if (count > 1) {
4472 struct res_counter *dummy;
4473 /*
4474 * "mem" cannot be under rmdir() because we've already checked
4475 * by cgroup_lock_live_cgroup() that it is not removed and we
4476 * are still under the same cgroup_mutex. So we can postpone
4477 * css_get().
4478 */
4479 if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
4480 goto one_by_one;
4481 if (do_swap_account && res_counter_charge(&mem->memsw,
4482 PAGE_SIZE * count, &dummy)) {
4483 res_counter_uncharge(&mem->res, PAGE_SIZE * count);
4484 goto one_by_one;
4485 }
4486 mc.precharge += count;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004487 return ret;
4488 }
4489one_by_one:
4490 /* fall back to one by one charge */
4491 while (count--) {
4492 if (signal_pending(current)) {
4493 ret = -EINTR;
4494 break;
4495 }
4496 if (!batch_count--) {
4497 batch_count = PRECHARGE_COUNT_AT_ONCE;
4498 cond_resched();
4499 }
Andrea Arcangeliec168512011-01-13 15:46:56 -08004500 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false,
4501 PAGE_SIZE);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004502 if (ret || !mem)
4503 /* mem_cgroup_clear_mc() will do uncharge later */
4504 return -ENOMEM;
4505 mc.precharge++;
4506 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004507 return ret;
4508}
4509
4510/**
4511 * is_target_pte_for_mc - check a pte whether it is valid for move charge
4512 * @vma: the vma the pte to be checked belongs
4513 * @addr: the address corresponding to the pte to be checked
4514 * @ptent: the pte to be checked
Daisuke Nishimura02491442010-03-10 15:22:17 -08004515 * @target: the pointer the target page or swap ent will be stored(can be NULL)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004516 *
4517 * Returns
4518 * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
4519 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4520 * move charge. if @target is not NULL, the page is stored in target->page
4521 * with extra refcnt got(Callers should handle it).
Daisuke Nishimura02491442010-03-10 15:22:17 -08004522 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4523 * target for charge migration. if @target is not NULL, the entry is stored
4524 * in target->ent.
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004525 *
4526 * Called with pte lock held.
4527 */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004528union mc_target {
4529 struct page *page;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004530 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004531};
4532
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004533enum mc_target_type {
4534 MC_TARGET_NONE, /* not used */
4535 MC_TARGET_PAGE,
Daisuke Nishimura02491442010-03-10 15:22:17 -08004536 MC_TARGET_SWAP,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004537};
4538
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004539static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4540 unsigned long addr, pte_t ptent)
4541{
4542 struct page *page = vm_normal_page(vma, addr, ptent);
4543
4544 if (!page || !page_mapped(page))
4545 return NULL;
4546 if (PageAnon(page)) {
4547 /* we don't move shared anon */
4548 if (!move_anon() || page_mapcount(page) > 2)
4549 return NULL;
Daisuke Nishimura87946a72010-05-26 14:42:39 -07004550 } else if (!move_file())
4551 /* we ignore mapcount for file pages */
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004552 return NULL;
4553 if (!get_page_unless_zero(page))
4554 return NULL;
4555
4556 return page;
4557}
4558
4559static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4560 unsigned long addr, pte_t ptent, swp_entry_t *entry)
4561{
4562 int usage_count;
4563 struct page *page = NULL;
4564 swp_entry_t ent = pte_to_swp_entry(ptent);
4565
4566 if (!move_anon() || non_swap_entry(ent))
4567 return NULL;
4568 usage_count = mem_cgroup_count_swap_user(ent, &page);
4569 if (usage_count > 1) { /* we don't move shared anon */
4570 if (page)
4571 put_page(page);
4572 return NULL;
4573 }
4574 if (do_swap_account)
4575 entry->val = ent.val;
4576
4577 return page;
4578}
4579
Daisuke Nishimura87946a72010-05-26 14:42:39 -07004580static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4581 unsigned long addr, pte_t ptent, swp_entry_t *entry)
4582{
4583 struct page *page = NULL;
4584 struct inode *inode;
4585 struct address_space *mapping;
4586 pgoff_t pgoff;
4587
4588 if (!vma->vm_file) /* anonymous vma */
4589 return NULL;
4590 if (!move_file())
4591 return NULL;
4592
4593 inode = vma->vm_file->f_path.dentry->d_inode;
4594 mapping = vma->vm_file->f_mapping;
4595 if (pte_none(ptent))
4596 pgoff = linear_page_index(vma, addr);
4597 else /* pte_file(ptent) is true */
4598 pgoff = pte_to_pgoff(ptent);
4599
4600 /* page is moved even if it's not RSS of this task(page-faulted). */
4601 if (!mapping_cap_swap_backed(mapping)) { /* normal file */
4602 page = find_get_page(mapping, pgoff);
4603 } else { /* shmem/tmpfs file. we should take account of swap too. */
4604 swp_entry_t ent;
4605 mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent);
4606 if (do_swap_account)
4607 entry->val = ent.val;
4608 }
4609
4610 return page;
4611}
4612
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004613static int is_target_pte_for_mc(struct vm_area_struct *vma,
4614 unsigned long addr, pte_t ptent, union mc_target *target)
4615{
Daisuke Nishimura02491442010-03-10 15:22:17 -08004616 struct page *page = NULL;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004617 struct page_cgroup *pc;
4618 int ret = 0;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004619 swp_entry_t ent = { .val = 0 };
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004620
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004621 if (pte_present(ptent))
4622 page = mc_handle_present_pte(vma, addr, ptent);
4623 else if (is_swap_pte(ptent))
4624 page = mc_handle_swap_pte(vma, addr, ptent, &ent);
Daisuke Nishimura87946a72010-05-26 14:42:39 -07004625 else if (pte_none(ptent) || pte_file(ptent))
4626 page = mc_handle_file_pte(vma, addr, ptent, &ent);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004627
4628 if (!page && !ent.val)
Daisuke Nishimura02491442010-03-10 15:22:17 -08004629 return 0;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004630 if (page) {
4631 pc = lookup_page_cgroup(page);
4632 /*
4633 * Do only loose check w/o page_cgroup lock.
4634 * mem_cgroup_move_account() checks the pc is valid or not under
4635 * the lock.
4636 */
4637 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
4638 ret = MC_TARGET_PAGE;
4639 if (target)
4640 target->page = page;
4641 }
4642 if (!ret || !target)
4643 put_page(page);
4644 }
Daisuke Nishimura90254a62010-05-26 14:42:38 -07004645 /* There is a swap entry and a page doesn't exist or isn't charged */
4646 if (ent.val && !ret &&
KAMEZAWA Hiroyuki7f0f1542010-05-11 14:06:58 -07004647 css_id(&mc.from->css) == lookup_swap_cgroup(ent)) {
4648 ret = MC_TARGET_SWAP;
4649 if (target)
4650 target->ent = ent;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004651 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004652 return ret;
4653}
4654
4655static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4656 unsigned long addr, unsigned long end,
4657 struct mm_walk *walk)
4658{
4659 struct vm_area_struct *vma = walk->private;
4660 pte_t *pte;
4661 spinlock_t *ptl;
4662
Andrea Arcangeliec168512011-01-13 15:46:56 -08004663 VM_BUG_ON(pmd_trans_huge(*pmd));
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004664 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4665 for (; addr != end; pte++, addr += PAGE_SIZE)
4666 if (is_target_pte_for_mc(vma, addr, *pte, NULL))
4667 mc.precharge++; /* increment precharge temporarily */
4668 pte_unmap_unlock(pte - 1, ptl);
4669 cond_resched();
4670
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004671 return 0;
4672}
4673
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004674static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4675{
4676 unsigned long precharge;
4677 struct vm_area_struct *vma;
4678
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004679 down_read(&mm->mmap_sem);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004680 for (vma = mm->mmap; vma; vma = vma->vm_next) {
4681 struct mm_walk mem_cgroup_count_precharge_walk = {
4682 .pmd_entry = mem_cgroup_count_precharge_pte_range,
4683 .mm = mm,
4684 .private = vma,
4685 };
4686 if (is_vm_hugetlb_page(vma))
4687 continue;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004688 walk_page_range(vma->vm_start, vma->vm_end,
4689 &mem_cgroup_count_precharge_walk);
4690 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004691 up_read(&mm->mmap_sem);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004692
4693 precharge = mc.precharge;
4694 mc.precharge = 0;
4695
4696 return precharge;
4697}
4698
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004699static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4700{
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004701 unsigned long precharge = mem_cgroup_count_precharge(mm);
4702
4703 VM_BUG_ON(mc.moving_task);
4704 mc.moving_task = current;
4705 return mem_cgroup_do_precharge(precharge);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004706}
4707
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004708/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
4709static void __mem_cgroup_clear_mc(void)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004710{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004711 struct mem_cgroup *from = mc.from;
4712 struct mem_cgroup *to = mc.to;
4713
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004714 /* we must uncharge all the leftover precharges from mc.to */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004715 if (mc.precharge) {
4716 __mem_cgroup_cancel_charge(mc.to, mc.precharge);
4717 mc.precharge = 0;
4718 }
4719 /*
4720 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4721 * we must uncharge here.
4722 */
4723 if (mc.moved_charge) {
4724 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
4725 mc.moved_charge = 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004726 }
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004727 /* we must fixup refcnts and charges */
4728 if (mc.moved_swap) {
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004729 /* uncharge swap account from the old cgroup */
4730 if (!mem_cgroup_is_root(mc.from))
4731 res_counter_uncharge(&mc.from->memsw,
4732 PAGE_SIZE * mc.moved_swap);
4733 __mem_cgroup_put(mc.from, mc.moved_swap);
4734
4735 if (!mem_cgroup_is_root(mc.to)) {
4736 /*
4737 * we charged both to->res and to->memsw, so we should
4738 * uncharge to->res.
4739 */
4740 res_counter_uncharge(&mc.to->res,
4741 PAGE_SIZE * mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004742 }
4743 /* we've already done mem_cgroup_get(mc.to) */
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004744 mc.moved_swap = 0;
4745 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004746 memcg_oom_recover(from);
4747 memcg_oom_recover(to);
4748 wake_up_all(&mc.waitq);
4749}
4750
4751static void mem_cgroup_clear_mc(void)
4752{
4753 struct mem_cgroup *from = mc.from;
4754
4755 /*
4756 * we must clear moving_task before waking up waiters at the end of
4757 * task migration.
4758 */
4759 mc.moving_task = NULL;
4760 __mem_cgroup_clear_mc();
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004761 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004762 mc.from = NULL;
4763 mc.to = NULL;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004764 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07004765 mem_cgroup_end_move(from);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004766}
4767
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004768static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
4769 struct cgroup *cgroup,
4770 struct task_struct *p,
4771 bool threadgroup)
4772{
4773 int ret = 0;
4774 struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup);
4775
4776 if (mem->move_charge_at_immigrate) {
4777 struct mm_struct *mm;
4778 struct mem_cgroup *from = mem_cgroup_from_task(p);
4779
4780 VM_BUG_ON(from == mem);
4781
4782 mm = get_task_mm(p);
4783 if (!mm)
4784 return 0;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004785 /* We move charges only when we move a owner of the mm */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004786 if (mm->owner == p) {
4787 VM_BUG_ON(mc.from);
4788 VM_BUG_ON(mc.to);
4789 VM_BUG_ON(mc.precharge);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004790 VM_BUG_ON(mc.moved_charge);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004791 VM_BUG_ON(mc.moved_swap);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07004792 mem_cgroup_start_move(from);
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004793 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004794 mc.from = from;
4795 mc.to = mem;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07004796 spin_unlock(&mc.lock);
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004797 /* We set mc.moving_task later */
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004798
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004799 ret = mem_cgroup_precharge_mc(mm);
4800 if (ret)
4801 mem_cgroup_clear_mc();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004802 }
4803 mmput(mm);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004804 }
4805 return ret;
4806}
4807
4808static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
4809 struct cgroup *cgroup,
4810 struct task_struct *p,
4811 bool threadgroup)
4812{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004813 mem_cgroup_clear_mc();
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004814}
4815
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004816static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4817 unsigned long addr, unsigned long end,
4818 struct mm_walk *walk)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004819{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004820 int ret = 0;
4821 struct vm_area_struct *vma = walk->private;
4822 pte_t *pte;
4823 spinlock_t *ptl;
4824
4825retry:
Andrea Arcangeliec168512011-01-13 15:46:56 -08004826 VM_BUG_ON(pmd_trans_huge(*pmd));
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004827 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4828 for (; addr != end; addr += PAGE_SIZE) {
4829 pte_t ptent = *(pte++);
4830 union mc_target target;
4831 int type;
4832 struct page *page;
4833 struct page_cgroup *pc;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004834 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004835
4836 if (!mc.precharge)
4837 break;
4838
4839 type = is_target_pte_for_mc(vma, addr, ptent, &target);
4840 switch (type) {
4841 case MC_TARGET_PAGE:
4842 page = target.page;
4843 if (isolate_lru_page(page))
4844 goto put;
4845 pc = lookup_page_cgroup(page);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004846 if (!mem_cgroup_move_account(pc,
4847 mc.from, mc.to, false)) {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004848 mc.precharge--;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004849 /* we uncharge from mc.from later. */
4850 mc.moved_charge++;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004851 }
4852 putback_lru_page(page);
4853put: /* is_target_pte_for_mc() gets the page */
4854 put_page(page);
4855 break;
Daisuke Nishimura02491442010-03-10 15:22:17 -08004856 case MC_TARGET_SWAP:
4857 ent = target.ent;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004858 if (!mem_cgroup_move_swap_account(ent,
4859 mc.from, mc.to, false)) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08004860 mc.precharge--;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004861 /* we fixup refcnts and charges later. */
4862 mc.moved_swap++;
4863 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08004864 break;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004865 default:
4866 break;
4867 }
4868 }
4869 pte_unmap_unlock(pte - 1, ptl);
4870 cond_resched();
4871
4872 if (addr != end) {
4873 /*
4874 * We have consumed all precharges we got in can_attach().
4875 * We try charge one by one, but don't do any additional
4876 * charges to mc.to if we have failed in charge once in attach()
4877 * phase.
4878 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08004879 ret = mem_cgroup_do_precharge(1);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004880 if (!ret)
4881 goto retry;
4882 }
4883
4884 return ret;
4885}
4886
4887static void mem_cgroup_move_charge(struct mm_struct *mm)
4888{
4889 struct vm_area_struct *vma;
4890
4891 lru_add_drain_all();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004892retry:
4893 if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
4894 /*
4895 * Someone who are holding the mmap_sem might be waiting in
4896 * waitq. So we cancel all extra charges, wake up all waiters,
4897 * and retry. Because we cancel precharges, we might not be able
4898 * to move enough charges, but moving charge is a best-effort
4899 * feature anyway, so it wouldn't be a big problem.
4900 */
4901 __mem_cgroup_clear_mc();
4902 cond_resched();
4903 goto retry;
4904 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004905 for (vma = mm->mmap; vma; vma = vma->vm_next) {
4906 int ret;
4907 struct mm_walk mem_cgroup_move_charge_walk = {
4908 .pmd_entry = mem_cgroup_move_charge_pte_range,
4909 .mm = mm,
4910 .private = vma,
4911 };
4912 if (is_vm_hugetlb_page(vma))
4913 continue;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004914 ret = walk_page_range(vma->vm_start, vma->vm_end,
4915 &mem_cgroup_move_charge_walk);
4916 if (ret)
4917 /*
4918 * means we have consumed all precharges and failed in
4919 * doing additional charge. Just abandon here.
4920 */
4921 break;
4922 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004923 up_read(&mm->mmap_sem);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004924}
4925
Balbir Singh67e465a2008-02-07 00:13:54 -08004926static void mem_cgroup_move_task(struct cgroup_subsys *ss,
4927 struct cgroup *cont,
4928 struct cgroup *old_cont,
Ben Blumbe367d02009-09-23 15:56:31 -07004929 struct task_struct *p,
4930 bool threadgroup)
Balbir Singh67e465a2008-02-07 00:13:54 -08004931{
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004932 struct mm_struct *mm;
4933
4934 if (!mc.to)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004935 /* no need to move charge */
4936 return;
4937
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08004938 mm = get_task_mm(p);
4939 if (mm) {
4940 mem_cgroup_move_charge(mm);
4941 mmput(mm);
4942 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08004943 mem_cgroup_clear_mc();
Balbir Singh67e465a2008-02-07 00:13:54 -08004944}
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07004945#else /* !CONFIG_MMU */
4946static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
4947 struct cgroup *cgroup,
4948 struct task_struct *p,
4949 bool threadgroup)
4950{
4951 return 0;
4952}
4953static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
4954 struct cgroup *cgroup,
4955 struct task_struct *p,
4956 bool threadgroup)
4957{
4958}
4959static void mem_cgroup_move_task(struct cgroup_subsys *ss,
4960 struct cgroup *cont,
4961 struct cgroup *old_cont,
4962 struct task_struct *p,
4963 bool threadgroup)
4964{
4965}
4966#endif
Balbir Singh67e465a2008-02-07 00:13:54 -08004967
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004968struct cgroup_subsys mem_cgroup_subsys = {
4969 .name = "memory",
4970 .subsys_id = mem_cgroup_subsys_id,
4971 .create = mem_cgroup_create,
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08004972 .pre_destroy = mem_cgroup_pre_destroy,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004973 .destroy = mem_cgroup_destroy,
4974 .populate = mem_cgroup_populate,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004975 .can_attach = mem_cgroup_can_attach,
4976 .cancel_attach = mem_cgroup_cancel_attach,
Balbir Singh67e465a2008-02-07 00:13:54 -08004977 .attach = mem_cgroup_move_task,
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004978 .early_init = 0,
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004979 .use_id = 1,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004980};
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004981
4982#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
Michal Hockoa42c3902010-11-24 12:57:08 -08004983static int __init enable_swap_account(char *s)
4984{
4985 /* consider enabled if no parameter or 1 is given */
4986 if (!s || !strcmp(s, "1"))
4987 really_do_swap_account = 1;
4988 else if (!strcmp(s, "0"))
4989 really_do_swap_account = 0;
4990 return 1;
4991}
4992__setup("swapaccount", enable_swap_account);
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004993
4994static int __init disable_swap_account(char *s)
4995{
Michal Hockoa42c3902010-11-24 12:57:08 -08004996 enable_swap_account("0");
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08004997 return 1;
4998}
4999__setup("noswapaccount", disable_swap_account);
5000#endif