blob: f4ec4e7ca4cd2b64f9cfb2ea8845eba90373e549 [file] [log] [blame]
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001/* memcontrol.c - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08006 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08009 * Memory thresholds
10 * Copyright (C) 2009 Nokia Corporation
11 * Author: Kirill A. Shutemov
12 *
Balbir Singh8cdea7c2008-02-07 00:13:50 -080013 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
17 *
18 * This program is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
22 */
23
24#include <linux/res_counter.h>
25#include <linux/memcontrol.h>
26#include <linux/cgroup.h>
Pavel Emelianov78fb7462008-02-07 00:13:51 -080027#include <linux/mm.h>
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -080028#include <linux/hugetlb.h>
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -080029#include <linux/pagemap.h>
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080030#include <linux/smp.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080031#include <linux/page-flags.h>
Balbir Singh66e17072008-02-07 00:13:56 -080032#include <linux/backing-dev.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080033#include <linux/bit_spinlock.h>
34#include <linux/rcupdate.h>
Balbir Singhe2224322009-04-02 16:57:39 -070035#include <linux/limits.h>
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -080036#include <linux/mutex.h>
Balbir Singhf64c3f52009-09-23 15:56:37 -070037#include <linux/rbtree.h>
Balbir Singhb6ac57d2008-04-29 01:00:19 -070038#include <linux/slab.h>
Balbir Singh66e17072008-02-07 00:13:56 -080039#include <linux/swap.h>
Daisuke Nishimura02491442010-03-10 15:22:17 -080040#include <linux/swapops.h>
Balbir Singh66e17072008-02-07 00:13:56 -080041#include <linux/spinlock.h>
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080042#include <linux/eventfd.h>
43#include <linux/sort.h>
Balbir Singh66e17072008-02-07 00:13:56 -080044#include <linux/fs.h>
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -080045#include <linux/seq_file.h>
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -070046#include <linux/vmalloc.h>
Christoph Lameterb69408e2008-10-18 20:26:14 -070047#include <linux/mm_inline.h>
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -070048#include <linux/page_cgroup.h>
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -080049#include <linux/cpu.h>
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -070050#include <linux/oom.h>
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -080051#include "internal.h"
Balbir Singh8cdea7c2008-02-07 00:13:50 -080052
Balbir Singh8697d332008-02-07 00:13:59 -080053#include <asm/uaccess.h>
54
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -070055#include <trace/events/vmscan.h>
56
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070057struct cgroup_subsys mem_cgroup_subsys __read_mostly;
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070058#define MEM_CGROUP_RECLAIM_RETRIES 5
Balbir Singh4b3bde42009-09-23 15:56:32 -070059struct mem_cgroup *root_mem_cgroup __read_mostly;
Balbir Singh8cdea7c2008-02-07 00:13:50 -080060
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080061#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
Li Zefan338c8432009-06-17 16:27:15 -070062/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080063int do_swap_account __read_mostly;
Michal Hockoa42c3902010-11-24 12:57:08 -080064
65/* for remember boot option*/
66#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED
67static int really_do_swap_account __initdata = 1;
68#else
69static int really_do_swap_account __initdata = 0;
70#endif
71
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080072#else
73#define do_swap_account (0)
74#endif
75
76
Balbir Singh8cdea7c2008-02-07 00:13:50 -080077/*
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080078 * Statistics for memory cgroup.
79 */
80enum mem_cgroup_stat_index {
81 /*
82 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
83 */
84 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
Balbir Singhd69b0422009-06-17 16:26:34 -070085 MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -080086 MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
Balbir Singh0c3e73e2009-09-23 15:56:42 -070087 MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -070088 MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -070089 MEM_CGROUP_ON_MOVE, /* someone is moving account between groups */
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080090 MEM_CGROUP_STAT_NSTATS,
91};
92
Johannes Weinere9f89742011-03-23 16:42:37 -070093enum mem_cgroup_events_index {
94 MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */
95 MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */
96 MEM_CGROUP_EVENTS_COUNT, /* # of pages paged in/out */
Ying Han456f9982011-05-26 16:25:38 -070097 MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */
98 MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */
Johannes Weinere9f89742011-03-23 16:42:37 -070099 MEM_CGROUP_EVENTS_NSTATS,
100};
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700101/*
102 * Per memcg event counter is incremented at every pagein/pageout. With THP,
103 * it will be incremated by the number of pages. This counter is used for
104 * for trigger some periodic events. This is straightforward and better
105 * than using jiffies etc. to handle periodic memcg event.
106 */
107enum mem_cgroup_events_target {
108 MEM_CGROUP_TARGET_THRESH,
109 MEM_CGROUP_TARGET_SOFTLIMIT,
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -0700110 MEM_CGROUP_TARGET_NUMAINFO,
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700111 MEM_CGROUP_NTARGETS,
112};
113#define THRESHOLDS_EVENTS_TARGET (128)
114#define SOFTLIMIT_EVENTS_TARGET (1024)
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -0700115#define NUMAINFO_EVENTS_TARGET (1024)
Johannes Weinere9f89742011-03-23 16:42:37 -0700116
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800117struct mem_cgroup_stat_cpu {
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700118 long count[MEM_CGROUP_STAT_NSTATS];
Johannes Weinere9f89742011-03-23 16:42:37 -0700119 unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700120 unsigned long targets[MEM_CGROUP_NTARGETS];
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800121};
122
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800123/*
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800124 * per-zone information in memory controller.
125 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800126struct mem_cgroup_per_zone {
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -0800127 /*
128 * spin_lock to protect the per cgroup LRU
129 */
Christoph Lameterb69408e2008-10-18 20:26:14 -0700130 struct list_head lists[NR_LRU_LISTS];
131 unsigned long count[NR_LRU_LISTS];
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -0800132
133 struct zone_reclaim_stat reclaim_stat;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700134 struct rb_node tree_node; /* RB tree node */
135 unsigned long long usage_in_excess;/* Set to the value by which */
136 /* the soft limit is exceeded*/
137 bool on_tree;
Balbir Singh4e416952009-09-23 15:56:39 -0700138 struct mem_cgroup *mem; /* Back pointer, we cannot */
139 /* use container_of */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800140};
141/* Macro for accessing counter */
142#define MEM_CGROUP_ZSTAT(mz, idx) ((mz)->count[(idx)])
143
144struct mem_cgroup_per_node {
145 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
146};
147
148struct mem_cgroup_lru_info {
149 struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
150};
151
152/*
Balbir Singhf64c3f52009-09-23 15:56:37 -0700153 * Cgroups above their limits are maintained in a RB-Tree, independent of
154 * their hierarchy representation
155 */
156
157struct mem_cgroup_tree_per_zone {
158 struct rb_root rb_root;
159 spinlock_t lock;
160};
161
162struct mem_cgroup_tree_per_node {
163 struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
164};
165
166struct mem_cgroup_tree {
167 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
168};
169
170static struct mem_cgroup_tree soft_limit_tree __read_mostly;
171
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800172struct mem_cgroup_threshold {
173 struct eventfd_ctx *eventfd;
174 u64 threshold;
175};
176
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700177/* For threshold */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800178struct mem_cgroup_threshold_ary {
179 /* An array index points to threshold just below usage. */
Phil Carmody5407a562010-05-26 14:42:42 -0700180 int current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800181 /* Size of entries[] */
182 unsigned int size;
183 /* Array of thresholds */
184 struct mem_cgroup_threshold entries[0];
185};
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700186
187struct mem_cgroup_thresholds {
188 /* Primary thresholds array */
189 struct mem_cgroup_threshold_ary *primary;
190 /*
191 * Spare threshold array.
192 * This is needed to make mem_cgroup_unregister_event() "never fail".
193 * It must be able to store at least primary->size - 1 entries.
194 */
195 struct mem_cgroup_threshold_ary *spare;
196};
197
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700198/* for OOM */
199struct mem_cgroup_eventfd_list {
200 struct list_head list;
201 struct eventfd_ctx *eventfd;
202};
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800203
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800204static void mem_cgroup_threshold(struct mem_cgroup *mem);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700205static void mem_cgroup_oom_notify(struct mem_cgroup *mem);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800206
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -0700207enum {
208 SCAN_BY_LIMIT,
209 SCAN_BY_SYSTEM,
210 NR_SCAN_CONTEXT,
211 SCAN_BY_SHRINK, /* not recorded now */
212};
213
214enum {
215 SCAN,
216 SCAN_ANON,
217 SCAN_FILE,
218 ROTATE,
219 ROTATE_ANON,
220 ROTATE_FILE,
221 FREED,
222 FREED_ANON,
223 FREED_FILE,
224 ELAPSED,
225 NR_SCANSTATS,
226};
227
228struct scanstat {
229 spinlock_t lock;
230 unsigned long stats[NR_SCAN_CONTEXT][NR_SCANSTATS];
231 unsigned long rootstats[NR_SCAN_CONTEXT][NR_SCANSTATS];
232};
233
234const char *scanstat_string[NR_SCANSTATS] = {
235 "scanned_pages",
236 "scanned_anon_pages",
237 "scanned_file_pages",
238 "rotated_pages",
239 "rotated_anon_pages",
240 "rotated_file_pages",
241 "freed_pages",
242 "freed_anon_pages",
243 "freed_file_pages",
244 "elapsed_ns",
245};
246#define SCANSTAT_WORD_LIMIT "_by_limit"
247#define SCANSTAT_WORD_SYSTEM "_by_system"
248#define SCANSTAT_WORD_HIERARCHY "_under_hierarchy"
249
250
Balbir Singhf64c3f52009-09-23 15:56:37 -0700251/*
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800252 * The memory controller data structure. The memory controller controls both
253 * page cache and RSS per cgroup. We would eventually like to provide
254 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
255 * to help the administrator determine what knobs to tune.
256 *
257 * TODO: Add a water mark for the memory controller. Reclaim will begin when
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800258 * we hit the water mark. May be even add a low water mark, such that
259 * no reclaim occurs from a cgroup at it's low water mark, this is
260 * a feature that will be implemented much later in the future.
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800261 */
262struct mem_cgroup {
263 struct cgroup_subsys_state css;
264 /*
265 * the counter to account for memory usage
266 */
267 struct res_counter res;
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800268 /*
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800269 * the counter to account for mem+swap usage.
270 */
271 struct res_counter memsw;
272 /*
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800273 * Per cgroup active and inactive list, similar to the
274 * per zone LRU lists.
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800275 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800276 struct mem_cgroup_lru_info info;
Balbir Singh6d61ef42009-01-07 18:08:06 -0800277 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -0200278 * While reclaiming in a hierarchy, we cache the last child we
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -0700279 * reclaimed from.
Balbir Singh6d61ef42009-01-07 18:08:06 -0800280 */
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -0700281 int last_scanned_child;
Ying Han889976d2011-05-26 16:25:33 -0700282 int last_scanned_node;
283#if MAX_NUMNODES > 1
284 nodemask_t scan_nodes;
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -0700285 atomic_t numainfo_events;
286 atomic_t numainfo_updating;
Ying Han889976d2011-05-26 16:25:33 -0700287#endif
Balbir Singh18f59ea2009-01-07 18:08:07 -0800288 /*
289 * Should the accounting and control be hierarchical, per subtree?
290 */
291 bool use_hierarchy;
Michal Hocko79dfdac2011-07-26 16:08:23 -0700292
293 bool oom_lock;
294 atomic_t under_oom;
295
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800296 atomic_t refcnt;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800297
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -0700298 int swappiness;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -0700299 /* OOM-Killer disable */
300 int oom_kill_disable;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -0800301
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -0700302 /* set when res.limit == memsw.limit */
303 bool memsw_is_minimum;
304
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800305 /* protect arrays of thresholds */
306 struct mutex thresholds_lock;
307
308 /* thresholds for memory usage. RCU-protected */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700309 struct mem_cgroup_thresholds thresholds;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -0700310
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800311 /* thresholds for mem+swap usage. RCU-protected */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700312 struct mem_cgroup_thresholds memsw_thresholds;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -0700313
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700314 /* For oom notifier event fd */
315 struct list_head oom_notify;
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -0700316 /* For recording LRU-scan statistics */
317 struct scanstat scanstat;
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800318 /*
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800319 * Should we move charges of a task when a task is moved into this
320 * mem_cgroup ? And what type of charges should we move ?
321 */
322 unsigned long move_charge_at_immigrate;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800323 /*
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800324 * percpu counter.
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800325 */
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800326 struct mem_cgroup_stat_cpu *stat;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700327 /*
328 * used when a cpu is offlined or other synchronizations
329 * See mem_cgroup_read_stat().
330 */
331 struct mem_cgroup_stat_cpu nocpu_base;
332 spinlock_t pcp_counter_lock;
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800333};
334
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800335/* Stuffs for move charges at task migration. */
336/*
337 * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
338 * left-shifted bitmap of these types.
339 */
340enum move_type {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800341 MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700342 MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800343 NR_MOVE_TYPE,
344};
345
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800346/* "mc" and its members are protected by cgroup_mutex */
347static struct move_charge_struct {
Daisuke Nishimurab1dd6932010-11-24 12:57:06 -0800348 spinlock_t lock; /* for from, to */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800349 struct mem_cgroup *from;
350 struct mem_cgroup *to;
351 unsigned long precharge;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -0800352 unsigned long moved_charge;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -0800353 unsigned long moved_swap;
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800354 struct task_struct *moving_task; /* a task moving charges */
355 wait_queue_head_t waitq; /* a waitq for other context */
356} mc = {
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -0700357 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800358 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
359};
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800360
Daisuke Nishimura90254a62010-05-26 14:42:38 -0700361static bool move_anon(void)
362{
363 return test_bit(MOVE_CHARGE_TYPE_ANON,
364 &mc.to->move_charge_at_immigrate);
365}
366
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700367static bool move_file(void)
368{
369 return test_bit(MOVE_CHARGE_TYPE_FILE,
370 &mc.to->move_charge_at_immigrate);
371}
372
Balbir Singh4e416952009-09-23 15:56:39 -0700373/*
374 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
375 * limit reclaim to prevent infinite loops, if they ever occur.
376 */
377#define MEM_CGROUP_MAX_RECLAIM_LOOPS (100)
378#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2)
379
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800380enum charge_type {
381 MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
382 MEM_CGROUP_CHARGE_TYPE_MAPPED,
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700383 MEM_CGROUP_CHARGE_TYPE_SHMEM, /* used by page migration of shmem */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700384 MEM_CGROUP_CHARGE_TYPE_FORCE, /* used by force_empty */
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -0800385 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -0700386 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700387 NR_CHARGE_TYPE,
388};
389
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800390/* for encoding cft->private value on file */
391#define _MEM (0)
392#define _MEMSWAP (1)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700393#define _OOM_TYPE (2)
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800394#define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
395#define MEMFILE_TYPE(val) (((val) >> 16) & 0xffff)
396#define MEMFILE_ATTR(val) ((val) & 0xffff)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700397/* Used for OOM nofiier */
398#define OOM_CONTROL (0)
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800399
Balbir Singh75822b42009-09-23 15:56:38 -0700400/*
401 * Reclaim flags for mem_cgroup_hierarchical_reclaim
402 */
403#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0
404#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
405#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1
406#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
Balbir Singh4e416952009-09-23 15:56:39 -0700407#define MEM_CGROUP_RECLAIM_SOFT_BIT 0x2
408#define MEM_CGROUP_RECLAIM_SOFT (1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
Balbir Singh75822b42009-09-23 15:56:38 -0700409
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800410static void mem_cgroup_get(struct mem_cgroup *mem);
411static void mem_cgroup_put(struct mem_cgroup *mem);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -0800412static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -0700413static void drain_all_stock_async(struct mem_cgroup *mem);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -0800414
Balbir Singhf64c3f52009-09-23 15:56:37 -0700415static struct mem_cgroup_per_zone *
416mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
417{
418 return &mem->info.nodeinfo[nid]->zoneinfo[zid];
419}
420
Wu Fengguangd3242362009-12-16 12:19:59 +0100421struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
422{
423 return &mem->css;
424}
425
Balbir Singhf64c3f52009-09-23 15:56:37 -0700426static struct mem_cgroup_per_zone *
Johannes Weiner97a6c372011-03-23 16:42:27 -0700427page_cgroup_zoneinfo(struct mem_cgroup *mem, struct page *page)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700428{
Johannes Weiner97a6c372011-03-23 16:42:27 -0700429 int nid = page_to_nid(page);
430 int zid = page_zonenum(page);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700431
Balbir Singhf64c3f52009-09-23 15:56:37 -0700432 return mem_cgroup_zoneinfo(mem, nid, zid);
433}
434
435static struct mem_cgroup_tree_per_zone *
436soft_limit_tree_node_zone(int nid, int zid)
437{
438 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
439}
440
441static struct mem_cgroup_tree_per_zone *
442soft_limit_tree_from_page(struct page *page)
443{
444 int nid = page_to_nid(page);
445 int zid = page_zonenum(page);
446
447 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
448}
449
450static void
Balbir Singh4e416952009-09-23 15:56:39 -0700451__mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
Balbir Singhf64c3f52009-09-23 15:56:37 -0700452 struct mem_cgroup_per_zone *mz,
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700453 struct mem_cgroup_tree_per_zone *mctz,
454 unsigned long long new_usage_in_excess)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700455{
456 struct rb_node **p = &mctz->rb_root.rb_node;
457 struct rb_node *parent = NULL;
458 struct mem_cgroup_per_zone *mz_node;
459
460 if (mz->on_tree)
461 return;
462
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700463 mz->usage_in_excess = new_usage_in_excess;
464 if (!mz->usage_in_excess)
465 return;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700466 while (*p) {
467 parent = *p;
468 mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
469 tree_node);
470 if (mz->usage_in_excess < mz_node->usage_in_excess)
471 p = &(*p)->rb_left;
472 /*
473 * We can't avoid mem cgroups that are over their soft
474 * limit by the same amount
475 */
476 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
477 p = &(*p)->rb_right;
478 }
479 rb_link_node(&mz->tree_node, parent, p);
480 rb_insert_color(&mz->tree_node, &mctz->rb_root);
481 mz->on_tree = true;
Balbir Singh4e416952009-09-23 15:56:39 -0700482}
483
484static void
485__mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
486 struct mem_cgroup_per_zone *mz,
487 struct mem_cgroup_tree_per_zone *mctz)
488{
489 if (!mz->on_tree)
490 return;
491 rb_erase(&mz->tree_node, &mctz->rb_root);
492 mz->on_tree = false;
493}
494
495static void
Balbir Singhf64c3f52009-09-23 15:56:37 -0700496mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
497 struct mem_cgroup_per_zone *mz,
498 struct mem_cgroup_tree_per_zone *mctz)
499{
500 spin_lock(&mctz->lock);
Balbir Singh4e416952009-09-23 15:56:39 -0700501 __mem_cgroup_remove_exceeded(mem, mz, mctz);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700502 spin_unlock(&mctz->lock);
503}
504
Balbir Singhf64c3f52009-09-23 15:56:37 -0700505
506static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
507{
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700508 unsigned long long excess;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700509 struct mem_cgroup_per_zone *mz;
510 struct mem_cgroup_tree_per_zone *mctz;
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700511 int nid = page_to_nid(page);
512 int zid = page_zonenum(page);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700513 mctz = soft_limit_tree_from_page(page);
514
515 /*
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700516 * Necessary to update all ancestors when hierarchy is used.
517 * because their event counter is not touched.
Balbir Singhf64c3f52009-09-23 15:56:37 -0700518 */
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700519 for (; mem; mem = parent_mem_cgroup(mem)) {
520 mz = mem_cgroup_zoneinfo(mem, nid, zid);
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700521 excess = res_counter_soft_limit_excess(&mem->res);
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700522 /*
523 * We have to update the tree if mz is on RB-tree or
524 * mem is over its softlimit.
525 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700526 if (excess || mz->on_tree) {
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700527 spin_lock(&mctz->lock);
528 /* if on-tree, remove it */
529 if (mz->on_tree)
530 __mem_cgroup_remove_exceeded(mem, mz, mctz);
531 /*
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700532 * Insert again. mz->usage_in_excess will be updated.
533 * If excess is 0, no tree ops.
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700534 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700535 __mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700536 spin_unlock(&mctz->lock);
537 }
Balbir Singhf64c3f52009-09-23 15:56:37 -0700538 }
539}
540
541static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
542{
543 int node, zone;
544 struct mem_cgroup_per_zone *mz;
545 struct mem_cgroup_tree_per_zone *mctz;
546
547 for_each_node_state(node, N_POSSIBLE) {
548 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
549 mz = mem_cgroup_zoneinfo(mem, node, zone);
550 mctz = soft_limit_tree_node_zone(node, zone);
551 mem_cgroup_remove_exceeded(mem, mz, mctz);
552 }
553 }
554}
555
Balbir Singh4e416952009-09-23 15:56:39 -0700556static struct mem_cgroup_per_zone *
557__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
558{
559 struct rb_node *rightmost = NULL;
KAMEZAWA Hiroyuki26251ea2009-10-01 15:44:08 -0700560 struct mem_cgroup_per_zone *mz;
Balbir Singh4e416952009-09-23 15:56:39 -0700561
562retry:
KAMEZAWA Hiroyuki26251ea2009-10-01 15:44:08 -0700563 mz = NULL;
Balbir Singh4e416952009-09-23 15:56:39 -0700564 rightmost = rb_last(&mctz->rb_root);
565 if (!rightmost)
566 goto done; /* Nothing to reclaim from */
567
568 mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
569 /*
570 * Remove the node now but someone else can add it back,
571 * we will to add it back at the end of reclaim to its correct
572 * position in the tree.
573 */
574 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
575 if (!res_counter_soft_limit_excess(&mz->mem->res) ||
576 !css_tryget(&mz->mem->css))
577 goto retry;
578done:
579 return mz;
580}
581
582static struct mem_cgroup_per_zone *
583mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
584{
585 struct mem_cgroup_per_zone *mz;
586
587 spin_lock(&mctz->lock);
588 mz = __mem_cgroup_largest_soft_limit_node(mctz);
589 spin_unlock(&mctz->lock);
590 return mz;
591}
592
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700593/*
594 * Implementation Note: reading percpu statistics for memcg.
595 *
596 * Both of vmstat[] and percpu_counter has threshold and do periodic
597 * synchronization to implement "quick" read. There are trade-off between
598 * reading cost and precision of value. Then, we may have a chance to implement
599 * a periodic synchronizion of counter in memcg's counter.
600 *
601 * But this _read() function is used for user interface now. The user accounts
602 * memory usage by memory cgroup and he _always_ requires exact value because
603 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
604 * have to visit all online cpus and make sum. So, for now, unnecessary
605 * synchronization is not implemented. (just implemented for cpu hotplug)
606 *
607 * If there are kernel internal actions which can make use of some not-exact
608 * value, and reading all cpu value can be performance bottleneck in some
609 * common workload, threashold and synchonization as vmstat[] should be
610 * implemented.
611 */
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700612static long mem_cgroup_read_stat(struct mem_cgroup *mem,
613 enum mem_cgroup_stat_index idx)
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800614{
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700615 long val = 0;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800616 int cpu;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800617
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700618 get_online_cpus();
619 for_each_online_cpu(cpu)
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800620 val += per_cpu(mem->stat->count[idx], cpu);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700621#ifdef CONFIG_HOTPLUG_CPU
622 spin_lock(&mem->pcp_counter_lock);
623 val += mem->nocpu_base.count[idx];
624 spin_unlock(&mem->pcp_counter_lock);
625#endif
626 put_online_cpus();
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800627 return val;
628}
629
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700630static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
631 bool charge)
632{
633 int val = (charge) ? 1 : -1;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800634 this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700635}
636
Ying Han456f9982011-05-26 16:25:38 -0700637void mem_cgroup_pgfault(struct mem_cgroup *mem, int val)
638{
639 this_cpu_add(mem->stat->events[MEM_CGROUP_EVENTS_PGFAULT], val);
640}
641
642void mem_cgroup_pgmajfault(struct mem_cgroup *mem, int val)
643{
644 this_cpu_add(mem->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT], val);
645}
646
Johannes Weinere9f89742011-03-23 16:42:37 -0700647static unsigned long mem_cgroup_read_events(struct mem_cgroup *mem,
648 enum mem_cgroup_events_index idx)
649{
650 unsigned long val = 0;
651 int cpu;
652
653 for_each_online_cpu(cpu)
654 val += per_cpu(mem->stat->events[idx], cpu);
655#ifdef CONFIG_HOTPLUG_CPU
656 spin_lock(&mem->pcp_counter_lock);
657 val += mem->nocpu_base.events[idx];
658 spin_unlock(&mem->pcp_counter_lock);
659#endif
660 return val;
661}
662
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700663static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800664 bool file, int nr_pages)
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800665{
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800666 preempt_disable();
667
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800668 if (file)
669 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], nr_pages);
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800670 else
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800671 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], nr_pages);
Balaji Rao55e462b2008-05-01 04:35:12 -0700672
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800673 /* pagein of a big page is an event. So, ignore page size */
674 if (nr_pages > 0)
Johannes Weinere9f89742011-03-23 16:42:37 -0700675 __this_cpu_inc(mem->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
KAMEZAWA Hiroyuki3751d602011-02-01 15:52:45 -0800676 else {
Johannes Weinere9f89742011-03-23 16:42:37 -0700677 __this_cpu_inc(mem->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
KAMEZAWA Hiroyuki3751d602011-02-01 15:52:45 -0800678 nr_pages = -nr_pages; /* for event */
679 }
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800680
Johannes Weinere9f89742011-03-23 16:42:37 -0700681 __this_cpu_add(mem->stat->events[MEM_CGROUP_EVENTS_COUNT], nr_pages);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800682
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800683 preempt_enable();
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800684}
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800685
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700686unsigned long
687mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *mem, int nid, int zid,
688 unsigned int lru_mask)
Ying Han889976d2011-05-26 16:25:33 -0700689{
690 struct mem_cgroup_per_zone *mz;
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700691 enum lru_list l;
692 unsigned long ret = 0;
693
694 mz = mem_cgroup_zoneinfo(mem, nid, zid);
695
696 for_each_lru(l) {
697 if (BIT(l) & lru_mask)
698 ret += MEM_CGROUP_ZSTAT(mz, l);
699 }
700 return ret;
701}
702
703static unsigned long
704mem_cgroup_node_nr_lru_pages(struct mem_cgroup *mem,
705 int nid, unsigned int lru_mask)
706{
Ying Han889976d2011-05-26 16:25:33 -0700707 u64 total = 0;
708 int zid;
709
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700710 for (zid = 0; zid < MAX_NR_ZONES; zid++)
711 total += mem_cgroup_zone_nr_lru_pages(mem, nid, zid, lru_mask);
712
Ying Han889976d2011-05-26 16:25:33 -0700713 return total;
714}
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700715
716static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *mem,
717 unsigned int lru_mask)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800718{
Ying Han889976d2011-05-26 16:25:33 -0700719 int nid;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800720 u64 total = 0;
721
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700722 for_each_node_state(nid, N_HIGH_MEMORY)
723 total += mem_cgroup_node_nr_lru_pages(mem, nid, lru_mask);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800724 return total;
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800725}
726
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700727static bool __memcg_event_check(struct mem_cgroup *mem, int target)
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800728{
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700729 unsigned long val, next;
730
731 val = this_cpu_read(mem->stat->events[MEM_CGROUP_EVENTS_COUNT]);
732 next = this_cpu_read(mem->stat->targets[target]);
733 /* from time_after() in jiffies.h */
734 return ((long)next - (long)val < 0);
735}
736
737static void __mem_cgroup_target_update(struct mem_cgroup *mem, int target)
738{
739 unsigned long val, next;
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800740
Johannes Weinere9f89742011-03-23 16:42:37 -0700741 val = this_cpu_read(mem->stat->events[MEM_CGROUP_EVENTS_COUNT]);
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800742
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700743 switch (target) {
744 case MEM_CGROUP_TARGET_THRESH:
745 next = val + THRESHOLDS_EVENTS_TARGET;
746 break;
747 case MEM_CGROUP_TARGET_SOFTLIMIT:
748 next = val + SOFTLIMIT_EVENTS_TARGET;
749 break;
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -0700750 case MEM_CGROUP_TARGET_NUMAINFO:
751 next = val + NUMAINFO_EVENTS_TARGET;
752 break;
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700753 default:
754 return;
755 }
756
757 this_cpu_write(mem->stat->targets[target], next);
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800758}
759
760/*
761 * Check events in order.
762 *
763 */
764static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
765{
766 /* threshold event is triggered in finer grain than soft limit */
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700767 if (unlikely(__memcg_event_check(mem, MEM_CGROUP_TARGET_THRESH))) {
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800768 mem_cgroup_threshold(mem);
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700769 __mem_cgroup_target_update(mem, MEM_CGROUP_TARGET_THRESH);
770 if (unlikely(__memcg_event_check(mem,
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -0700771 MEM_CGROUP_TARGET_SOFTLIMIT))) {
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800772 mem_cgroup_update_tree(mem, page);
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700773 __mem_cgroup_target_update(mem,
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -0700774 MEM_CGROUP_TARGET_SOFTLIMIT);
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700775 }
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -0700776#if MAX_NUMNODES > 1
777 if (unlikely(__memcg_event_check(mem,
778 MEM_CGROUP_TARGET_NUMAINFO))) {
779 atomic_inc(&mem->numainfo_events);
780 __mem_cgroup_target_update(mem,
781 MEM_CGROUP_TARGET_NUMAINFO);
782 }
783#endif
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800784 }
785}
786
Hugh Dickinsd5b69e32008-03-04 14:29:10 -0800787static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800788{
789 return container_of(cgroup_subsys_state(cont,
790 mem_cgroup_subsys_id), struct mem_cgroup,
791 css);
792}
793
Balbir Singhcf475ad2008-04-29 01:00:16 -0700794struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800795{
Balbir Singh31a78f22008-09-28 23:09:31 +0100796 /*
797 * mm_update_next_owner() may clear mm->owner to NULL
798 * if it races with swapoff, page migration, etc.
799 * So this can be called with p == NULL.
800 */
801 if (unlikely(!p))
802 return NULL;
803
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800804 return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
805 struct mem_cgroup, css);
806}
807
KOSAKI Motohiroa4336582011-06-15 15:08:13 -0700808struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800809{
810 struct mem_cgroup *mem = NULL;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -0700811
812 if (!mm)
813 return NULL;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -0800814 /*
815 * Because we have no locks, mm->owner's may be being moved to other
816 * cgroup. We use css_tryget() here even if this looks
817 * pessimistic (rather than adding locks here).
818 */
819 rcu_read_lock();
820 do {
821 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
822 if (unlikely(!mem))
823 break;
824 } while (!css_tryget(&mem->css));
825 rcu_read_unlock();
826 return mem;
827}
828
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700829/* The caller has to guarantee "mem" exists before calling this */
830static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700831{
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700832 struct cgroup_subsys_state *css;
833 int found;
834
835 if (!mem) /* ROOT cgroup has the smallest ID */
836 return root_mem_cgroup; /*css_put/get against root is ignored*/
837 if (!mem->use_hierarchy) {
838 if (css_tryget(&mem->css))
839 return mem;
840 return NULL;
841 }
842 rcu_read_lock();
843 /*
844 * searching a memory cgroup which has the smallest ID under given
845 * ROOT cgroup. (ID >= 1)
846 */
847 css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found);
848 if (css && css_tryget(css))
849 mem = container_of(css, struct mem_cgroup, css);
850 else
851 mem = NULL;
852 rcu_read_unlock();
853 return mem;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -0700854}
855
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700856static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter,
857 struct mem_cgroup *root,
858 bool cond)
859{
860 int nextid = css_id(&iter->css) + 1;
861 int found;
862 int hierarchy_used;
863 struct cgroup_subsys_state *css;
864
865 hierarchy_used = iter->use_hierarchy;
866
867 css_put(&iter->css);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700868 /* If no ROOT, walk all, ignore hierarchy */
869 if (!cond || (root && !hierarchy_used))
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700870 return NULL;
871
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700872 if (!root)
873 root = root_mem_cgroup;
874
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700875 do {
876 iter = NULL;
877 rcu_read_lock();
878
879 css = css_get_next(&mem_cgroup_subsys, nextid,
880 &root->css, &found);
881 if (css && css_tryget(css))
882 iter = container_of(css, struct mem_cgroup, css);
883 rcu_read_unlock();
884 /* If css is NULL, no more cgroups will be found */
885 nextid = found + 1;
886 } while (css && !iter);
887
888 return iter;
889}
890/*
891 * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please
892 * be careful that "break" loop is not allowed. We have reference count.
893 * Instead of that modify "cond" to be false and "continue" to exit the loop.
894 */
895#define for_each_mem_cgroup_tree_cond(iter, root, cond) \
896 for (iter = mem_cgroup_start_loop(root);\
897 iter != NULL;\
898 iter = mem_cgroup_get_next(iter, root, cond))
899
900#define for_each_mem_cgroup_tree(iter, root) \
901 for_each_mem_cgroup_tree_cond(iter, root, true)
902
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700903#define for_each_mem_cgroup_all(iter) \
904 for_each_mem_cgroup_tree_cond(iter, NULL, true)
905
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -0700906
Balbir Singh4b3bde42009-09-23 15:56:32 -0700907static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
908{
909 return (mem == root_mem_cgroup);
910}
911
Ying Han456f9982011-05-26 16:25:38 -0700912void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
913{
914 struct mem_cgroup *mem;
915
916 if (!mm)
917 return;
918
919 rcu_read_lock();
920 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
921 if (unlikely(!mem))
922 goto out;
923
924 switch (idx) {
925 case PGMAJFAULT:
926 mem_cgroup_pgmajfault(mem, 1);
927 break;
928 case PGFAULT:
929 mem_cgroup_pgfault(mem, 1);
930 break;
931 default:
932 BUG();
933 }
934out:
935 rcu_read_unlock();
936}
937EXPORT_SYMBOL(mem_cgroup_count_vm_event);
938
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800939/*
940 * Following LRU functions are allowed to be used without PCG_LOCK.
941 * Operations are called by routine of global LRU independently from memcg.
942 * What we have to take care of here is validness of pc->mem_cgroup.
943 *
944 * Changes to pc->mem_cgroup happens when
945 * 1. charge
946 * 2. moving account
947 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
948 * It is added to LRU before charge.
949 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
950 * When moving account, the page is not on LRU. It's isolated.
951 */
952
953void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800954{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800955 struct page_cgroup *pc;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800956 struct mem_cgroup_per_zone *mz;
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700957
Hirokazu Takahashif8d66542009-01-07 18:08:02 -0800958 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800959 return;
960 pc = lookup_page_cgroup(page);
961 /* can happen while we handle swapcache. */
Balbir Singh4b3bde42009-09-23 15:56:32 -0700962 if (!TestClearPageCgroupAcctLRU(pc))
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800963 return;
Balbir Singh4b3bde42009-09-23 15:56:32 -0700964 VM_BUG_ON(!pc->mem_cgroup);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -0800965 /*
966 * We don't check PCG_USED bit. It's cleared when the "page" is finally
967 * removed from global LRU.
968 */
Johannes Weiner97a6c372011-03-23 16:42:27 -0700969 mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
KAMEZAWA Hiroyukiece35ca2011-01-20 14:44:24 -0800970 /* huge page split is done under lru_lock. so, we have no races. */
971 MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page);
Balbir Singh4b3bde42009-09-23 15:56:32 -0700972 if (mem_cgroup_is_root(pc->mem_cgroup))
973 return;
974 VM_BUG_ON(list_empty(&pc->lru));
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800975 list_del_init(&pc->lru);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800976}
977
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800978void mem_cgroup_del_lru(struct page *page)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800979{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -0800980 mem_cgroup_del_lru_list(page, page_lru(page));
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800981}
982
Minchan Kim3f58a822011-03-22 16:32:53 -0700983/*
984 * Writeback is about to end against a page which has been marked for immediate
985 * reclaim. If it still appears to be reclaimable, move it to the tail of the
986 * inactive list.
987 */
988void mem_cgroup_rotate_reclaimable_page(struct page *page)
989{
990 struct mem_cgroup_per_zone *mz;
991 struct page_cgroup *pc;
992 enum lru_list lru = page_lru(page);
993
994 if (mem_cgroup_disabled())
995 return;
996
997 pc = lookup_page_cgroup(page);
998 /* unused or root page is not rotated. */
999 if (!PageCgroupUsed(pc))
1000 return;
1001 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
1002 smp_rmb();
1003 if (mem_cgroup_is_root(pc->mem_cgroup))
1004 return;
Johannes Weiner97a6c372011-03-23 16:42:27 -07001005 mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
Minchan Kim3f58a822011-03-22 16:32:53 -07001006 list_move_tail(&pc->lru, &mz->lists[lru]);
1007}
1008
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001009void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
Balbir Singh66e17072008-02-07 00:13:56 -08001010{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001011 struct mem_cgroup_per_zone *mz;
1012 struct page_cgroup *pc;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001013
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08001014 if (mem_cgroup_disabled())
Lee Schermerhorn894bc312008-10-18 20:26:39 -07001015 return;
Christoph Lameterb69408e2008-10-18 20:26:14 -07001016
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001017 pc = lookup_page_cgroup(page);
Balbir Singh4b3bde42009-09-23 15:56:32 -07001018 /* unused or root page is not rotated. */
Johannes Weiner713735b2011-01-20 14:44:31 -08001019 if (!PageCgroupUsed(pc))
1020 return;
1021 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
1022 smp_rmb();
1023 if (mem_cgroup_is_root(pc->mem_cgroup))
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001024 return;
Johannes Weiner97a6c372011-03-23 16:42:27 -07001025 mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
Christoph Lameterb69408e2008-10-18 20:26:14 -07001026 list_move(&pc->lru, &mz->lists[lru]);
Balbir Singh66e17072008-02-07 00:13:56 -08001027}
1028
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001029void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
1030{
1031 struct page_cgroup *pc;
1032 struct mem_cgroup_per_zone *mz;
1033
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08001034 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001035 return;
1036 pc = lookup_page_cgroup(page);
Balbir Singh4b3bde42009-09-23 15:56:32 -07001037 VM_BUG_ON(PageCgroupAcctLRU(pc));
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001038 if (!PageCgroupUsed(pc))
1039 return;
Johannes Weiner713735b2011-01-20 14:44:31 -08001040 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
1041 smp_rmb();
Johannes Weiner97a6c372011-03-23 16:42:27 -07001042 mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
KAMEZAWA Hiroyukiece35ca2011-01-20 14:44:24 -08001043 /* huge page split is done under lru_lock. so, we have no races. */
1044 MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page);
Balbir Singh4b3bde42009-09-23 15:56:32 -07001045 SetPageCgroupAcctLRU(pc);
1046 if (mem_cgroup_is_root(pc->mem_cgroup))
1047 return;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001048 list_add(&pc->lru, &mz->lists[lru]);
1049}
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08001050
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001051/*
KAMEZAWA Hiroyuki5a6475a2011-03-23 16:42:42 -07001052 * At handling SwapCache and other FUSE stuff, pc->mem_cgroup may be changed
1053 * while it's linked to lru because the page may be reused after it's fully
1054 * uncharged. To handle that, unlink page_cgroup from LRU when charge it again.
1055 * It's done under lock_page and expected that zone->lru_lock isnever held.
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001056 */
KAMEZAWA Hiroyuki5a6475a2011-03-23 16:42:42 -07001057static void mem_cgroup_lru_del_before_commit(struct page *page)
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001058{
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08001059 unsigned long flags;
1060 struct zone *zone = page_zone(page);
1061 struct page_cgroup *pc = lookup_page_cgroup(page);
1062
KAMEZAWA Hiroyuki5a6475a2011-03-23 16:42:42 -07001063 /*
1064 * Doing this check without taking ->lru_lock seems wrong but this
1065 * is safe. Because if page_cgroup's USED bit is unset, the page
1066 * will not be added to any memcg's LRU. If page_cgroup's USED bit is
1067 * set, the commit after this will fail, anyway.
1068 * This all charge/uncharge is done under some mutual execustion.
1069 * So, we don't need to taking care of changes in USED bit.
1070 */
1071 if (likely(!PageLRU(page)))
1072 return;
1073
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08001074 spin_lock_irqsave(&zone->lru_lock, flags);
1075 /*
1076 * Forget old LRU when this page_cgroup is *not* used. This Used bit
1077 * is guarded by lock_page() because the page is SwapCache.
1078 */
1079 if (!PageCgroupUsed(pc))
1080 mem_cgroup_del_lru_list(page, page_lru(page));
1081 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001082}
1083
KAMEZAWA Hiroyuki5a6475a2011-03-23 16:42:42 -07001084static void mem_cgroup_lru_add_after_commit(struct page *page)
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08001085{
1086 unsigned long flags;
1087 struct zone *zone = page_zone(page);
1088 struct page_cgroup *pc = lookup_page_cgroup(page);
1089
KAMEZAWA Hiroyuki5a6475a2011-03-23 16:42:42 -07001090 /* taking care of that the page is added to LRU while we commit it */
1091 if (likely(!PageLRU(page)))
1092 return;
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08001093 spin_lock_irqsave(&zone->lru_lock, flags);
1094 /* link when the page is linked to LRU but page_cgroup isn't */
Balbir Singh4b3bde42009-09-23 15:56:32 -07001095 if (PageLRU(page) && !PageCgroupAcctLRU(pc))
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08001096 mem_cgroup_add_lru_list(page, page_lru(page));
1097 spin_unlock_irqrestore(&zone->lru_lock, flags);
1098}
1099
1100
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001101void mem_cgroup_move_lists(struct page *page,
1102 enum lru_list from, enum lru_list to)
1103{
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08001104 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001105 return;
1106 mem_cgroup_del_lru_list(page, from);
1107 mem_cgroup_add_lru_list(page, to);
1108}
1109
Michal Hocko3e920412011-07-26 16:08:29 -07001110/*
1111 * Checks whether given mem is same or in the root_mem's
1112 * hierarchy subtree
1113 */
1114static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_mem,
1115 struct mem_cgroup *mem)
1116{
1117 if (root_mem != mem) {
1118 return (root_mem->use_hierarchy &&
1119 css_is_ancestor(&mem->css, &root_mem->css));
1120 }
1121
1122 return true;
1123}
1124
David Rientjes4c4a2212008-02-07 00:14:06 -08001125int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
1126{
1127 int ret;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001128 struct mem_cgroup *curr = NULL;
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -07001129 struct task_struct *p;
David Rientjes4c4a2212008-02-07 00:14:06 -08001130
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -07001131 p = find_lock_task_mm(task);
1132 if (!p)
1133 return 0;
1134 curr = try_get_mem_cgroup_from_mm(p->mm);
1135 task_unlock(p);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001136 if (!curr)
1137 return 0;
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -08001138 /*
1139 * We should check use_hierarchy of "mem" not "curr". Because checking
1140 * use_hierarchy of "curr" here make this function true if hierarchy is
1141 * enabled in "curr" and "curr" is a child of "mem" in *cgroup*
1142 * hierarchy(even if use_hierarchy is disabled in "mem").
1143 */
Michal Hocko3e920412011-07-26 16:08:29 -07001144 ret = mem_cgroup_same_or_subtree(mem, curr);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001145 css_put(&curr->css);
David Rientjes4c4a2212008-02-07 00:14:06 -08001146 return ret;
1147}
1148
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08001149static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001150{
1151 unsigned long active;
1152 unsigned long inactive;
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08001153 unsigned long gb;
1154 unsigned long inactive_ratio;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001155
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07001156 inactive = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_ANON));
1157 active = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_ANON));
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001158
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08001159 gb = (inactive + active) >> (30 - PAGE_SHIFT);
1160 if (gb)
1161 inactive_ratio = int_sqrt(10 * gb);
1162 else
1163 inactive_ratio = 1;
1164
1165 if (present_pages) {
1166 present_pages[0] = inactive;
1167 present_pages[1] = active;
1168 }
1169
1170 return inactive_ratio;
1171}
1172
1173int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
1174{
1175 unsigned long active;
1176 unsigned long inactive;
1177 unsigned long present_pages[2];
1178 unsigned long inactive_ratio;
1179
1180 inactive_ratio = calc_inactive_ratio(memcg, present_pages);
1181
1182 inactive = present_pages[0];
1183 active = present_pages[1];
1184
1185 if (inactive * inactive_ratio < active)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001186 return 1;
1187
1188 return 0;
1189}
1190
Rik van Riel56e49d22009-06-16 15:32:28 -07001191int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
1192{
1193 unsigned long active;
1194 unsigned long inactive;
1195
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07001196 inactive = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_FILE));
1197 active = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_FILE));
Rik van Riel56e49d22009-06-16 15:32:28 -07001198
1199 return (active > inactive);
1200}
1201
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -08001202struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
1203 struct zone *zone)
1204{
KOSAKI Motohiro13d7e3a2010-08-10 18:03:06 -07001205 int nid = zone_to_nid(zone);
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -08001206 int zid = zone_idx(zone);
1207 struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
1208
1209 return &mz->reclaim_stat;
1210}
1211
1212struct zone_reclaim_stat *
1213mem_cgroup_get_reclaim_stat_from_page(struct page *page)
1214{
1215 struct page_cgroup *pc;
1216 struct mem_cgroup_per_zone *mz;
1217
1218 if (mem_cgroup_disabled())
1219 return NULL;
1220
1221 pc = lookup_page_cgroup(page);
Daisuke Nishimurabd112db2009-01-15 13:51:11 -08001222 if (!PageCgroupUsed(pc))
1223 return NULL;
Johannes Weiner713735b2011-01-20 14:44:31 -08001224 /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
1225 smp_rmb();
Johannes Weiner97a6c372011-03-23 16:42:27 -07001226 mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -08001227 return &mz->reclaim_stat;
1228}
1229
Balbir Singh66e17072008-02-07 00:13:56 -08001230unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
1231 struct list_head *dst,
1232 unsigned long *scanned, int order,
1233 int mode, struct zone *z,
1234 struct mem_cgroup *mem_cont,
Rik van Riel4f98a2f2008-10-18 20:26:32 -07001235 int active, int file)
Balbir Singh66e17072008-02-07 00:13:56 -08001236{
1237 unsigned long nr_taken = 0;
1238 struct page *page;
1239 unsigned long scan;
1240 LIST_HEAD(pc_list);
1241 struct list_head *src;
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001242 struct page_cgroup *pc, *tmp;
KOSAKI Motohiro13d7e3a2010-08-10 18:03:06 -07001243 int nid = zone_to_nid(z);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001244 int zid = zone_idx(z);
1245 struct mem_cgroup_per_zone *mz;
Johannes Weinerb7c46d12009-09-21 17:02:56 -07001246 int lru = LRU_FILE * file + active;
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001247 int ret;
Balbir Singh66e17072008-02-07 00:13:56 -08001248
Balbir Singhcf475ad2008-04-29 01:00:16 -07001249 BUG_ON(!mem_cont);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08001250 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
Christoph Lameterb69408e2008-10-18 20:26:14 -07001251 src = &mz->lists[lru];
Balbir Singh66e17072008-02-07 00:13:56 -08001252
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001253 scan = 0;
1254 list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
Hugh Dickins436c65412008-02-07 00:14:12 -08001255 if (scan >= nr_to_scan)
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001256 break;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001257
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07001258 if (unlikely(!PageCgroupUsed(pc)))
1259 continue;
Johannes Weiner5564e882011-03-23 16:42:29 -07001260
Johannes Weiner6b3ae582011-03-23 16:42:30 -07001261 page = lookup_cgroup_page(pc);
Johannes Weiner5564e882011-03-23 16:42:29 -07001262
Hugh Dickins436c65412008-02-07 00:14:12 -08001263 if (unlikely(!PageLRU(page)))
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001264 continue;
KAMEZAWA Hiroyukiff7283f2008-02-07 00:14:11 -08001265
Hugh Dickins436c65412008-02-07 00:14:12 -08001266 scan++;
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001267 ret = __isolate_lru_page(page, mode, file);
1268 switch (ret) {
1269 case 0:
Balbir Singh66e17072008-02-07 00:13:56 -08001270 list_move(&page->lru, dst);
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001271 mem_cgroup_del_lru(page);
Rik van Riel2c888cf2011-01-13 15:47:13 -08001272 nr_taken += hpage_nr_pages(page);
KAMEZAWA Hiroyuki2ffebca2009-06-17 16:27:21 -07001273 break;
1274 case -EBUSY:
1275 /* we don't affect global LRU but rotate in our LRU */
1276 mem_cgroup_rotate_lru_list(page, page_lru(page));
1277 break;
1278 default:
1279 break;
Balbir Singh66e17072008-02-07 00:13:56 -08001280 }
1281 }
1282
Balbir Singh66e17072008-02-07 00:13:56 -08001283 *scanned = scan;
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -07001284
1285 trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken,
1286 0, 0, 0, mode);
1287
Balbir Singh66e17072008-02-07 00:13:56 -08001288 return nr_taken;
1289}
1290
Balbir Singh6d61ef42009-01-07 18:08:06 -08001291#define mem_cgroup_from_res_counter(counter, member) \
1292 container_of(counter, struct mem_cgroup, member)
1293
Johannes Weiner19942822011-02-01 15:52:43 -08001294/**
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001295 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
1296 * @mem: the memory cgroup
Johannes Weiner19942822011-02-01 15:52:43 -08001297 *
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001298 * Returns the maximum amount of memory @mem can be charged with, in
Johannes Weiner7ec99d62011-03-23 16:42:36 -07001299 * pages.
Johannes Weiner19942822011-02-01 15:52:43 -08001300 */
Johannes Weiner7ec99d62011-03-23 16:42:36 -07001301static unsigned long mem_cgroup_margin(struct mem_cgroup *mem)
Johannes Weiner19942822011-02-01 15:52:43 -08001302{
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001303 unsigned long long margin;
1304
1305 margin = res_counter_margin(&mem->res);
1306 if (do_swap_account)
1307 margin = min(margin, res_counter_margin(&mem->memsw));
Johannes Weiner7ec99d62011-03-23 16:42:36 -07001308 return margin >> PAGE_SHIFT;
Johannes Weiner19942822011-02-01 15:52:43 -08001309}
1310
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07001311int mem_cgroup_swappiness(struct mem_cgroup *memcg)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001312{
1313 struct cgroup *cgrp = memcg->css.cgroup;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001314
1315 /* root ? */
1316 if (cgrp->parent == NULL)
1317 return vm_swappiness;
1318
Johannes Weinerbf1ff262011-03-23 16:42:32 -07001319 return memcg->swappiness;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001320}
1321
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001322static void mem_cgroup_start_move(struct mem_cgroup *mem)
1323{
1324 int cpu;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001325
1326 get_online_cpus();
1327 spin_lock(&mem->pcp_counter_lock);
1328 for_each_online_cpu(cpu)
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001329 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001330 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
1331 spin_unlock(&mem->pcp_counter_lock);
1332 put_online_cpus();
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001333
1334 synchronize_rcu();
1335}
1336
1337static void mem_cgroup_end_move(struct mem_cgroup *mem)
1338{
1339 int cpu;
1340
1341 if (!mem)
1342 return;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001343 get_online_cpus();
1344 spin_lock(&mem->pcp_counter_lock);
1345 for_each_online_cpu(cpu)
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001346 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07001347 mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
1348 spin_unlock(&mem->pcp_counter_lock);
1349 put_online_cpus();
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001350}
1351/*
1352 * 2 routines for checking "mem" is under move_account() or not.
1353 *
1354 * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used
1355 * for avoiding race in accounting. If true,
1356 * pc->mem_cgroup may be overwritten.
1357 *
1358 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
1359 * under hierarchy of moving cgroups. This is for
1360 * waiting at hith-memory prressure caused by "move".
1361 */
1362
1363static bool mem_cgroup_stealed(struct mem_cgroup *mem)
1364{
1365 VM_BUG_ON(!rcu_read_lock_held());
1366 return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
1367}
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001368
1369static bool mem_cgroup_under_move(struct mem_cgroup *mem)
1370{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001371 struct mem_cgroup *from;
1372 struct mem_cgroup *to;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001373 bool ret = false;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001374 /*
1375 * Unlike task_move routines, we access mc.to, mc.from not under
1376 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1377 */
1378 spin_lock(&mc.lock);
1379 from = mc.from;
1380 to = mc.to;
1381 if (!from)
1382 goto unlock;
Michal Hocko3e920412011-07-26 16:08:29 -07001383
1384 ret = mem_cgroup_same_or_subtree(mem, from)
1385 || mem_cgroup_same_or_subtree(mem, to);
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001386unlock:
1387 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001388 return ret;
1389}
1390
1391static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem)
1392{
1393 if (mc.moving_task && current != mc.moving_task) {
1394 if (mem_cgroup_under_move(mem)) {
1395 DEFINE_WAIT(wait);
1396 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1397 /* moving charge context might have finished. */
1398 if (mc.moving_task)
1399 schedule();
1400 finish_wait(&mc.waitq, &wait);
1401 return true;
1402 }
1403 }
1404 return false;
1405}
1406
Balbir Singhe2224322009-04-02 16:57:39 -07001407/**
Kirill A. Shutemov6a6135b2010-03-10 15:22:25 -08001408 * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
Balbir Singhe2224322009-04-02 16:57:39 -07001409 * @memcg: The memory cgroup that went over limit
1410 * @p: Task that is going to be killed
1411 *
1412 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1413 * enabled
1414 */
1415void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1416{
1417 struct cgroup *task_cgrp;
1418 struct cgroup *mem_cgrp;
1419 /*
1420 * Need a buffer in BSS, can't rely on allocations. The code relies
1421 * on the assumption that OOM is serialized for memory controller.
1422 * If this assumption is broken, revisit this code.
1423 */
1424 static char memcg_name[PATH_MAX];
1425 int ret;
1426
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -08001427 if (!memcg || !p)
Balbir Singhe2224322009-04-02 16:57:39 -07001428 return;
1429
1430
1431 rcu_read_lock();
1432
1433 mem_cgrp = memcg->css.cgroup;
1434 task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1435
1436 ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1437 if (ret < 0) {
1438 /*
1439 * Unfortunately, we are unable to convert to a useful name
1440 * But we'll still print out the usage information
1441 */
1442 rcu_read_unlock();
1443 goto done;
1444 }
1445 rcu_read_unlock();
1446
1447 printk(KERN_INFO "Task in %s killed", memcg_name);
1448
1449 rcu_read_lock();
1450 ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1451 if (ret < 0) {
1452 rcu_read_unlock();
1453 goto done;
1454 }
1455 rcu_read_unlock();
1456
1457 /*
1458 * Continues from above, so we don't need an KERN_ level
1459 */
1460 printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
1461done:
1462
1463 printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
1464 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1465 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1466 res_counter_read_u64(&memcg->res, RES_FAILCNT));
1467 printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
1468 "failcnt %llu\n",
1469 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1470 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1471 res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1472}
1473
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001474/*
1475 * This function returns the number of memcg under hierarchy tree. Returns
1476 * 1(self count) if no children.
1477 */
1478static int mem_cgroup_count_children(struct mem_cgroup *mem)
1479{
1480 int num = 0;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001481 struct mem_cgroup *iter;
1482
1483 for_each_mem_cgroup_tree(iter, mem)
1484 num++;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001485 return num;
1486}
1487
Balbir Singh6d61ef42009-01-07 18:08:06 -08001488/*
David Rientjesa63d83f2010-08-09 17:19:46 -07001489 * Return the memory (and swap, if configured) limit for a memcg.
1490 */
1491u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
1492{
1493 u64 limit;
1494 u64 memsw;
1495
Johannes Weinerf3e8eb72011-01-13 15:47:39 -08001496 limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
1497 limit += total_swap_pages << PAGE_SHIFT;
1498
David Rientjesa63d83f2010-08-09 17:19:46 -07001499 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1500 /*
1501 * If memsw is finite and limits the amount of swap space available
1502 * to this memcg, return that limit.
1503 */
1504 return min(limit, memsw);
1505}
1506
1507/*
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001508 * Visit the first child (need not be the first child as per the ordering
1509 * of the cgroup list, since we track last_scanned_child) of @mem and use
1510 * that to reclaim free pages from.
1511 */
1512static struct mem_cgroup *
1513mem_cgroup_select_victim(struct mem_cgroup *root_mem)
1514{
1515 struct mem_cgroup *ret = NULL;
1516 struct cgroup_subsys_state *css;
1517 int nextid, found;
1518
1519 if (!root_mem->use_hierarchy) {
1520 css_get(&root_mem->css);
1521 ret = root_mem;
1522 }
1523
1524 while (!ret) {
1525 rcu_read_lock();
1526 nextid = root_mem->last_scanned_child + 1;
1527 css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
1528 &found);
1529 if (css && css_tryget(css))
1530 ret = container_of(css, struct mem_cgroup, css);
1531
1532 rcu_read_unlock();
1533 /* Updates scanning parameter */
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001534 if (!css) {
1535 /* this means start scan from ID:1 */
1536 root_mem->last_scanned_child = 0;
1537 } else
1538 root_mem->last_scanned_child = found;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001539 }
1540
1541 return ret;
1542}
1543
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001544/**
1545 * test_mem_cgroup_node_reclaimable
1546 * @mem: the target memcg
1547 * @nid: the node ID to be checked.
1548 * @noswap : specify true here if the user wants flle only information.
1549 *
1550 * This function returns whether the specified memcg contains any
1551 * reclaimable pages on a node. Returns true if there are any reclaimable
1552 * pages in the node.
1553 */
1554static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *mem,
1555 int nid, bool noswap)
1556{
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07001557 if (mem_cgroup_node_nr_lru_pages(mem, nid, LRU_ALL_FILE))
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001558 return true;
1559 if (noswap || !total_swap_pages)
1560 return false;
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07001561 if (mem_cgroup_node_nr_lru_pages(mem, nid, LRU_ALL_ANON))
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001562 return true;
1563 return false;
1564
1565}
Ying Han889976d2011-05-26 16:25:33 -07001566#if MAX_NUMNODES > 1
1567
1568/*
1569 * Always updating the nodemask is not very good - even if we have an empty
1570 * list or the wrong list here, we can start from some node and traverse all
1571 * nodes based on the zonelist. So update the list loosely once per 10 secs.
1572 *
1573 */
1574static void mem_cgroup_may_update_nodemask(struct mem_cgroup *mem)
1575{
1576 int nid;
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -07001577 /*
1578 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1579 * pagein/pageout changes since the last update.
1580 */
1581 if (!atomic_read(&mem->numainfo_events))
1582 return;
1583 if (atomic_inc_return(&mem->numainfo_updating) > 1)
Ying Han889976d2011-05-26 16:25:33 -07001584 return;
1585
Ying Han889976d2011-05-26 16:25:33 -07001586 /* make a nodemask where this memcg uses memory from */
1587 mem->scan_nodes = node_states[N_HIGH_MEMORY];
1588
1589 for_each_node_mask(nid, node_states[N_HIGH_MEMORY]) {
1590
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001591 if (!test_mem_cgroup_node_reclaimable(mem, nid, false))
1592 node_clear(nid, mem->scan_nodes);
Ying Han889976d2011-05-26 16:25:33 -07001593 }
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -07001594
1595 atomic_set(&mem->numainfo_events, 0);
1596 atomic_set(&mem->numainfo_updating, 0);
Ying Han889976d2011-05-26 16:25:33 -07001597}
1598
1599/*
1600 * Selecting a node where we start reclaim from. Because what we need is just
1601 * reducing usage counter, start from anywhere is O,K. Considering
1602 * memory reclaim from current node, there are pros. and cons.
1603 *
1604 * Freeing memory from current node means freeing memory from a node which
1605 * we'll use or we've used. So, it may make LRU bad. And if several threads
1606 * hit limits, it will see a contention on a node. But freeing from remote
1607 * node means more costs for memory reclaim because of memory latency.
1608 *
1609 * Now, we use round-robin. Better algorithm is welcomed.
1610 */
1611int mem_cgroup_select_victim_node(struct mem_cgroup *mem)
1612{
1613 int node;
1614
1615 mem_cgroup_may_update_nodemask(mem);
1616 node = mem->last_scanned_node;
1617
1618 node = next_node(node, mem->scan_nodes);
1619 if (node == MAX_NUMNODES)
1620 node = first_node(mem->scan_nodes);
1621 /*
1622 * We call this when we hit limit, not when pages are added to LRU.
1623 * No LRU may hold pages because all pages are UNEVICTABLE or
1624 * memcg is too small and all pages are not on LRU. In that case,
1625 * we use curret node.
1626 */
1627 if (unlikely(node == MAX_NUMNODES))
1628 node = numa_node_id();
1629
1630 mem->last_scanned_node = node;
1631 return node;
1632}
1633
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001634/*
1635 * Check all nodes whether it contains reclaimable pages or not.
1636 * For quick scan, we make use of scan_nodes. This will allow us to skip
1637 * unused nodes. But scan_nodes is lazily updated and may not cotain
1638 * enough new information. We need to do double check.
1639 */
1640bool mem_cgroup_reclaimable(struct mem_cgroup *mem, bool noswap)
1641{
1642 int nid;
1643
1644 /*
1645 * quick check...making use of scan_node.
1646 * We can skip unused nodes.
1647 */
1648 if (!nodes_empty(mem->scan_nodes)) {
1649 for (nid = first_node(mem->scan_nodes);
1650 nid < MAX_NUMNODES;
1651 nid = next_node(nid, mem->scan_nodes)) {
1652
1653 if (test_mem_cgroup_node_reclaimable(mem, nid, noswap))
1654 return true;
1655 }
1656 }
1657 /*
1658 * Check rest of nodes.
1659 */
1660 for_each_node_state(nid, N_HIGH_MEMORY) {
1661 if (node_isset(nid, mem->scan_nodes))
1662 continue;
1663 if (test_mem_cgroup_node_reclaimable(mem, nid, noswap))
1664 return true;
1665 }
1666 return false;
1667}
1668
Ying Han889976d2011-05-26 16:25:33 -07001669#else
1670int mem_cgroup_select_victim_node(struct mem_cgroup *mem)
1671{
1672 return 0;
1673}
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001674
1675bool mem_cgroup_reclaimable(struct mem_cgroup *mem, bool noswap)
1676{
1677 return test_mem_cgroup_node_reclaimable(mem, 0, noswap);
1678}
Ying Han889976d2011-05-26 16:25:33 -07001679#endif
1680
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07001681static void __mem_cgroup_record_scanstat(unsigned long *stats,
1682 struct memcg_scanrecord *rec)
1683{
1684
1685 stats[SCAN] += rec->nr_scanned[0] + rec->nr_scanned[1];
1686 stats[SCAN_ANON] += rec->nr_scanned[0];
1687 stats[SCAN_FILE] += rec->nr_scanned[1];
1688
1689 stats[ROTATE] += rec->nr_rotated[0] + rec->nr_rotated[1];
1690 stats[ROTATE_ANON] += rec->nr_rotated[0];
1691 stats[ROTATE_FILE] += rec->nr_rotated[1];
1692
1693 stats[FREED] += rec->nr_freed[0] + rec->nr_freed[1];
1694 stats[FREED_ANON] += rec->nr_freed[0];
1695 stats[FREED_FILE] += rec->nr_freed[1];
1696
1697 stats[ELAPSED] += rec->elapsed;
1698}
1699
1700static void mem_cgroup_record_scanstat(struct memcg_scanrecord *rec)
1701{
1702 struct mem_cgroup *mem;
1703 int context = rec->context;
1704
1705 if (context >= NR_SCAN_CONTEXT)
1706 return;
1707
1708 mem = rec->mem;
1709 spin_lock(&mem->scanstat.lock);
1710 __mem_cgroup_record_scanstat(mem->scanstat.stats[context], rec);
1711 spin_unlock(&mem->scanstat.lock);
1712
1713 mem = rec->root;
1714 spin_lock(&mem->scanstat.lock);
1715 __mem_cgroup_record_scanstat(mem->scanstat.rootstats[context], rec);
1716 spin_unlock(&mem->scanstat.lock);
1717}
1718
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001719/*
1720 * Scan the hierarchy if needed to reclaim memory. We remember the last child
1721 * we reclaimed from, so that we don't end up penalizing one child extensively
1722 * based on its position in the children list.
Balbir Singh6d61ef42009-01-07 18:08:06 -08001723 *
1724 * root_mem is the original ancestor that we've been reclaim from.
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001725 *
1726 * We give up and return to the caller when we visit root_mem twice.
1727 * (other groups can be removed while we're walking....)
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001728 *
1729 * If shrink==true, for avoiding to free too much, this returns immedieately.
Balbir Singh6d61ef42009-01-07 18:08:06 -08001730 */
1731static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
Balbir Singh4e416952009-09-23 15:56:39 -07001732 struct zone *zone,
Balbir Singh75822b42009-09-23 15:56:38 -07001733 gfp_t gfp_mask,
Ying Han0ae5e892011-05-26 16:25:25 -07001734 unsigned long reclaim_options,
1735 unsigned long *total_scanned)
Balbir Singh6d61ef42009-01-07 18:08:06 -08001736{
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001737 struct mem_cgroup *victim;
1738 int ret, total = 0;
1739 int loop = 0;
Balbir Singh75822b42009-09-23 15:56:38 -07001740 bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
1741 bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
Balbir Singh4e416952009-09-23 15:56:39 -07001742 bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07001743 struct memcg_scanrecord rec;
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001744 unsigned long excess;
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07001745 unsigned long scanned;
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001746
1747 excess = res_counter_soft_limit_excess(&root_mem->res) >> PAGE_SHIFT;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001748
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07001749 /* If memsw_is_minimum==1, swap-out is of-no-use. */
Daisuke Nishimura108b6a72011-07-26 16:08:25 -07001750 if (!check_soft && !shrink && root_mem->memsw_is_minimum)
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07001751 noswap = true;
1752
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07001753 if (shrink)
1754 rec.context = SCAN_BY_SHRINK;
1755 else if (check_soft)
1756 rec.context = SCAN_BY_SYSTEM;
1757 else
1758 rec.context = SCAN_BY_LIMIT;
1759
1760 rec.root = root_mem;
1761
Balbir Singh4e416952009-09-23 15:56:39 -07001762 while (1) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001763 victim = mem_cgroup_select_victim(root_mem);
Balbir Singh4e416952009-09-23 15:56:39 -07001764 if (victim == root_mem) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001765 loop++;
KAMEZAWA Hiroyukifbc29a22011-06-15 15:08:46 -07001766 /*
1767 * We are not draining per cpu cached charges during
1768 * soft limit reclaim because global reclaim doesn't
1769 * care about charges. It tries to free some memory and
1770 * charges will not give any.
1771 */
1772 if (!check_soft && loop >= 1)
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07001773 drain_all_stock_async(root_mem);
Balbir Singh4e416952009-09-23 15:56:39 -07001774 if (loop >= 2) {
1775 /*
1776 * If we have not been able to reclaim
1777 * anything, it might because there are
1778 * no reclaimable pages under this hierarchy
1779 */
1780 if (!check_soft || !total) {
1781 css_put(&victim->css);
1782 break;
1783 }
1784 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001785 * We want to do more targeted reclaim.
Balbir Singh4e416952009-09-23 15:56:39 -07001786 * excess >> 2 is not to excessive so as to
1787 * reclaim too much, nor too less that we keep
1788 * coming back to reclaim from this cgroup
1789 */
1790 if (total >= (excess >> 2) ||
1791 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
1792 css_put(&victim->css);
1793 break;
1794 }
1795 }
1796 }
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001797 if (!mem_cgroup_reclaimable(victim, noswap)) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001798 /* this cgroup's local usage == 0 */
1799 css_put(&victim->css);
Balbir Singh6d61ef42009-01-07 18:08:06 -08001800 continue;
1801 }
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07001802 rec.mem = victim;
1803 rec.nr_scanned[0] = 0;
1804 rec.nr_scanned[1] = 0;
1805 rec.nr_rotated[0] = 0;
1806 rec.nr_rotated[1] = 0;
1807 rec.nr_freed[0] = 0;
1808 rec.nr_freed[1] = 0;
1809 rec.elapsed = 0;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001810 /* we use swappiness of local cgroup */
Ying Han0ae5e892011-05-26 16:25:25 -07001811 if (check_soft) {
Balbir Singh4e416952009-09-23 15:56:39 -07001812 ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07001813 noswap, zone, &rec, &scanned);
1814 *total_scanned += scanned;
Ying Han0ae5e892011-05-26 16:25:25 -07001815 } else
Balbir Singh4e416952009-09-23 15:56:39 -07001816 ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07001817 noswap, &rec);
1818 mem_cgroup_record_scanstat(&rec);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001819 css_put(&victim->css);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001820 /*
1821 * At shrinking usage, we can't check we should stop here or
1822 * reclaim more. It's depends on callers. last_scanned_child
1823 * will work enough for keeping fairness under tree.
1824 */
1825 if (shrink)
1826 return ret;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001827 total += ret;
Balbir Singh4e416952009-09-23 15:56:39 -07001828 if (check_soft) {
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001829 if (!res_counter_soft_limit_excess(&root_mem->res))
Balbir Singh4e416952009-09-23 15:56:39 -07001830 return total;
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001831 } else if (mem_cgroup_margin(root_mem))
Johannes Weiner4fd14eb2011-05-26 16:25:35 -07001832 return total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001833 }
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001834 return total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001835}
1836
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001837/*
1838 * Check OOM-Killer is already running under our hierarchy.
1839 * If someone is running, return false.
Michal Hocko1af8efe2011-07-26 16:08:24 -07001840 * Has to be called with memcg_oom_lock
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001841 */
1842static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
1843{
Michal Hocko79dfdac2011-07-26 16:08:23 -07001844 int lock_count = -1;
1845 struct mem_cgroup *iter, *failed = NULL;
1846 bool cond = true;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001847
Michal Hocko79dfdac2011-07-26 16:08:23 -07001848 for_each_mem_cgroup_tree_cond(iter, mem, cond) {
1849 bool locked = iter->oom_lock;
1850
1851 iter->oom_lock = true;
1852 if (lock_count == -1)
1853 lock_count = iter->oom_lock;
1854 else if (lock_count != locked) {
1855 /*
1856 * this subtree of our hierarchy is already locked
1857 * so we cannot give a lock.
1858 */
1859 lock_count = 0;
1860 failed = iter;
1861 cond = false;
1862 }
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001863 }
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001864
Michal Hocko79dfdac2011-07-26 16:08:23 -07001865 if (!failed)
1866 goto done;
1867
1868 /*
1869 * OK, we failed to lock the whole subtree so we have to clean up
1870 * what we set up to the failing subtree
1871 */
1872 cond = true;
1873 for_each_mem_cgroup_tree_cond(iter, mem, cond) {
1874 if (iter == failed) {
1875 cond = false;
1876 continue;
1877 }
1878 iter->oom_lock = false;
1879 }
1880done:
1881 return lock_count;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001882}
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001883
Michal Hocko79dfdac2011-07-26 16:08:23 -07001884/*
Michal Hocko1af8efe2011-07-26 16:08:24 -07001885 * Has to be called with memcg_oom_lock
Michal Hocko79dfdac2011-07-26 16:08:23 -07001886 */
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001887static int mem_cgroup_oom_unlock(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001888{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001889 struct mem_cgroup *iter;
1890
Michal Hocko79dfdac2011-07-26 16:08:23 -07001891 for_each_mem_cgroup_tree(iter, mem)
1892 iter->oom_lock = false;
1893 return 0;
1894}
1895
1896static void mem_cgroup_mark_under_oom(struct mem_cgroup *mem)
1897{
1898 struct mem_cgroup *iter;
1899
1900 for_each_mem_cgroup_tree(iter, mem)
1901 atomic_inc(&iter->under_oom);
1902}
1903
1904static void mem_cgroup_unmark_under_oom(struct mem_cgroup *mem)
1905{
1906 struct mem_cgroup *iter;
1907
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001908 /*
1909 * When a new child is created while the hierarchy is under oom,
1910 * mem_cgroup_oom_lock() may not be called. We have to use
1911 * atomic_add_unless() here.
1912 */
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001913 for_each_mem_cgroup_tree(iter, mem)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001914 atomic_add_unless(&iter->under_oom, -1, 0);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001915}
1916
Michal Hocko1af8efe2011-07-26 16:08:24 -07001917static DEFINE_SPINLOCK(memcg_oom_lock);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001918static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1919
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001920struct oom_wait_info {
1921 struct mem_cgroup *mem;
1922 wait_queue_t wait;
1923};
1924
1925static int memcg_oom_wake_function(wait_queue_t *wait,
1926 unsigned mode, int sync, void *arg)
1927{
Michal Hocko3e920412011-07-26 16:08:29 -07001928 struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg,
1929 *oom_wait_mem;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001930 struct oom_wait_info *oom_wait_info;
1931
1932 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
Michal Hocko3e920412011-07-26 16:08:29 -07001933 oom_wait_mem = oom_wait_info->mem;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001934
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001935 /*
1936 * Both of oom_wait_info->mem and wake_mem are stable under us.
1937 * Then we can use css_is_ancestor without taking care of RCU.
1938 */
Michal Hocko3e920412011-07-26 16:08:29 -07001939 if (!mem_cgroup_same_or_subtree(oom_wait_mem, wake_mem)
1940 && !mem_cgroup_same_or_subtree(wake_mem, oom_wait_mem))
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001941 return 0;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001942 return autoremove_wake_function(wait, mode, sync, arg);
1943}
1944
1945static void memcg_wakeup_oom(struct mem_cgroup *mem)
1946{
1947 /* for filtering, pass "mem" as argument. */
1948 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem);
1949}
1950
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001951static void memcg_oom_recover(struct mem_cgroup *mem)
1952{
Michal Hocko79dfdac2011-07-26 16:08:23 -07001953 if (mem && atomic_read(&mem->under_oom))
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001954 memcg_wakeup_oom(mem);
1955}
1956
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001957/*
1958 * try to call OOM killer. returns false if we should exit memory-reclaim loop.
1959 */
1960bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
1961{
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001962 struct oom_wait_info owait;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001963 bool locked, need_to_kill;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001964
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001965 owait.mem = mem;
1966 owait.wait.flags = 0;
1967 owait.wait.func = memcg_oom_wake_function;
1968 owait.wait.private = current;
1969 INIT_LIST_HEAD(&owait.wait.task_list);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001970 need_to_kill = true;
Michal Hocko79dfdac2011-07-26 16:08:23 -07001971 mem_cgroup_mark_under_oom(mem);
1972
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001973 /* At first, try to OOM lock hierarchy under mem.*/
Michal Hocko1af8efe2011-07-26 16:08:24 -07001974 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001975 locked = mem_cgroup_oom_lock(mem);
1976 /*
1977 * Even if signal_pending(), we can't quit charge() loop without
1978 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
1979 * under OOM is always welcomed, use TASK_KILLABLE here.
1980 */
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001981 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1982 if (!locked || mem->oom_kill_disable)
1983 need_to_kill = false;
1984 if (locked)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07001985 mem_cgroup_oom_notify(mem);
Michal Hocko1af8efe2011-07-26 16:08:24 -07001986 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001987
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001988 if (need_to_kill) {
1989 finish_wait(&memcg_oom_waitq, &owait.wait);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001990 mem_cgroup_out_of_memory(mem, mask);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07001991 } else {
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001992 schedule();
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001993 finish_wait(&memcg_oom_waitq, &owait.wait);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001994 }
Michal Hocko1af8efe2011-07-26 16:08:24 -07001995 spin_lock(&memcg_oom_lock);
Michal Hocko79dfdac2011-07-26 16:08:23 -07001996 if (locked)
1997 mem_cgroup_oom_unlock(mem);
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07001998 memcg_wakeup_oom(mem);
Michal Hocko1af8efe2011-07-26 16:08:24 -07001999 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002000
Michal Hocko79dfdac2011-07-26 16:08:23 -07002001 mem_cgroup_unmark_under_oom(mem);
2002
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002003 if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
2004 return false;
2005 /* Give chance to dying process */
2006 schedule_timeout(1);
2007 return true;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07002008}
2009
Balbir Singhd69b0422009-06-17 16:26:34 -07002010/*
2011 * Currently used to update mapped file statistics, but the routine can be
2012 * generalized to update other statistics as well.
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002013 *
2014 * Notes: Race condition
2015 *
2016 * We usually use page_cgroup_lock() for accessing page_cgroup member but
2017 * it tends to be costly. But considering some conditions, we doesn't need
2018 * to do so _always_.
2019 *
2020 * Considering "charge", lock_page_cgroup() is not required because all
2021 * file-stat operations happen after a page is attached to radix-tree. There
2022 * are no race with "charge".
2023 *
2024 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
2025 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
2026 * if there are race with "uncharge". Statistics itself is properly handled
2027 * by flags.
2028 *
2029 * Considering "move", this is an only case we see a race. To make the race
2030 * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are
2031 * possibility of race condition. If there is, we take a lock.
Balbir Singhd69b0422009-06-17 16:26:34 -07002032 */
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002033
Greg Thelen2a7106f2011-01-13 15:47:37 -08002034void mem_cgroup_update_page_stat(struct page *page,
2035 enum mem_cgroup_page_stat_item idx, int val)
Balbir Singhd69b0422009-06-17 16:26:34 -07002036{
2037 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002038 struct page_cgroup *pc = lookup_page_cgroup(page);
2039 bool need_unlock = false;
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08002040 unsigned long uninitialized_var(flags);
Balbir Singhd69b0422009-06-17 16:26:34 -07002041
Balbir Singhd69b0422009-06-17 16:26:34 -07002042 if (unlikely(!pc))
2043 return;
2044
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002045 rcu_read_lock();
Balbir Singhd69b0422009-06-17 16:26:34 -07002046 mem = pc->mem_cgroup;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002047 if (unlikely(!mem || !PageCgroupUsed(pc)))
2048 goto out;
2049 /* pc->mem_cgroup is unstable ? */
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002050 if (unlikely(mem_cgroup_stealed(mem)) || PageTransHuge(page)) {
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002051 /* take a lock against to access pc->mem_cgroup */
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08002052 move_lock_page_cgroup(pc, &flags);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002053 need_unlock = true;
2054 mem = pc->mem_cgroup;
2055 if (!mem || !PageCgroupUsed(pc))
2056 goto out;
2057 }
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002058
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002059 switch (idx) {
Greg Thelen2a7106f2011-01-13 15:47:37 -08002060 case MEMCG_NR_FILE_MAPPED:
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002061 if (val > 0)
2062 SetPageCgroupFileMapped(pc);
2063 else if (!page_mapped(page))
KAMEZAWA Hiroyuki0c270f82010-10-27 15:33:39 -07002064 ClearPageCgroupFileMapped(pc);
Greg Thelen2a7106f2011-01-13 15:47:37 -08002065 idx = MEM_CGROUP_STAT_FILE_MAPPED;
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002066 break;
2067 default:
2068 BUG();
KAMEZAWA Hiroyuki8725d542010-04-06 14:35:05 -07002069 }
Balbir Singhd69b0422009-06-17 16:26:34 -07002070
Greg Thelen2a7106f2011-01-13 15:47:37 -08002071 this_cpu_add(mem->stat->count[idx], val);
2072
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002073out:
2074 if (unlikely(need_unlock))
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08002075 move_unlock_page_cgroup(pc, &flags);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002076 rcu_read_unlock();
2077 return;
Balbir Singhd69b0422009-06-17 16:26:34 -07002078}
Greg Thelen2a7106f2011-01-13 15:47:37 -08002079EXPORT_SYMBOL(mem_cgroup_update_page_stat);
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002080
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002081/*
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002082 * size of first charge trial. "32" comes from vmscan.c's magic value.
2083 * TODO: maybe necessary to use big numbers in big irons.
2084 */
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002085#define CHARGE_BATCH 32U
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002086struct memcg_stock_pcp {
2087 struct mem_cgroup *cached; /* this never be root cgroup */
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002088 unsigned int nr_pages;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002089 struct work_struct work;
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002090 unsigned long flags;
2091#define FLUSHING_CACHED_CHARGE (0)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002092};
2093static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002094
2095/*
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002096 * Try to consume stocked charge on this cpu. If success, one page is consumed
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002097 * from local stock and true is returned. If the stock is 0 or charges from a
2098 * cgroup which is not current target, returns false. This stock will be
2099 * refilled.
2100 */
2101static bool consume_stock(struct mem_cgroup *mem)
2102{
2103 struct memcg_stock_pcp *stock;
2104 bool ret = true;
2105
2106 stock = &get_cpu_var(memcg_stock);
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002107 if (mem == stock->cached && stock->nr_pages)
2108 stock->nr_pages--;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002109 else /* need to call res_counter_charge */
2110 ret = false;
2111 put_cpu_var(memcg_stock);
2112 return ret;
2113}
2114
2115/*
2116 * Returns stocks cached in percpu to res_counter and reset cached information.
2117 */
2118static void drain_stock(struct memcg_stock_pcp *stock)
2119{
2120 struct mem_cgroup *old = stock->cached;
2121
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002122 if (stock->nr_pages) {
2123 unsigned long bytes = stock->nr_pages * PAGE_SIZE;
2124
2125 res_counter_uncharge(&old->res, bytes);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002126 if (do_swap_account)
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002127 res_counter_uncharge(&old->memsw, bytes);
2128 stock->nr_pages = 0;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002129 }
2130 stock->cached = NULL;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002131}
2132
2133/*
2134 * This must be called under preempt disabled or must be called by
2135 * a thread which is pinned to local cpu.
2136 */
2137static void drain_local_stock(struct work_struct *dummy)
2138{
2139 struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
2140 drain_stock(stock);
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002141 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002142}
2143
2144/*
2145 * Cache charges(val) which is from res_counter, to local per_cpu area.
Greg Thelen320cc512010-03-15 15:27:28 +01002146 * This will be consumed by consume_stock() function, later.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002147 */
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002148static void refill_stock(struct mem_cgroup *mem, unsigned int nr_pages)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002149{
2150 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
2151
2152 if (stock->cached != mem) { /* reset if necessary */
2153 drain_stock(stock);
2154 stock->cached = mem;
2155 }
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002156 stock->nr_pages += nr_pages;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002157 put_cpu_var(memcg_stock);
2158}
2159
2160/*
Michal Hockod38144b2011-07-26 16:08:28 -07002161 * Drains all per-CPU charge caches for given root_mem resp. subtree
2162 * of the hierarchy under it. sync flag says whether we should block
2163 * until the work is done.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002164 */
Michal Hockod38144b2011-07-26 16:08:28 -07002165static void drain_all_stock(struct mem_cgroup *root_mem, bool sync)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002166{
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002167 int cpu, curcpu;
Michal Hockod38144b2011-07-26 16:08:28 -07002168
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002169 /* Notify other cpus that system-wide "drain" is running */
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002170 get_online_cpus();
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002171 /*
2172 * Get a hint for avoiding draining charges on the current cpu,
2173 * which must be exhausted by our charging. It is not required that
2174 * this be a precise check, so we use raw_smp_processor_id() instead of
2175 * getcpu()/putcpu().
2176 */
2177 curcpu = raw_smp_processor_id();
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002178 for_each_online_cpu(cpu) {
2179 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002180 struct mem_cgroup *mem;
2181
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002182 mem = stock->cached;
Michal Hockod1a05b62011-07-26 16:08:27 -07002183 if (!mem || !stock->nr_pages)
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002184 continue;
Michal Hocko3e920412011-07-26 16:08:29 -07002185 if (!mem_cgroup_same_or_subtree(root_mem, mem))
2186 continue;
Michal Hockod1a05b62011-07-26 16:08:27 -07002187 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2188 if (cpu == curcpu)
2189 drain_local_stock(&stock->work);
2190 else
2191 schedule_work_on(cpu, &stock->work);
2192 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002193 }
Michal Hockod38144b2011-07-26 16:08:28 -07002194
2195 if (!sync)
2196 goto out;
2197
2198 for_each_online_cpu(cpu) {
2199 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
Michal Hocko8521fc52011-07-26 16:08:29 -07002200 if (mem_cgroup_same_or_subtree(root_mem, stock->cached) &&
2201 test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
Michal Hockod38144b2011-07-26 16:08:28 -07002202 flush_work(&stock->work);
2203 }
2204out:
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002205 put_online_cpus();
Michal Hockod38144b2011-07-26 16:08:28 -07002206}
2207
2208/*
2209 * Tries to drain stocked charges in other cpus. This function is asynchronous
2210 * and just put a work per cpu for draining localy on each cpu. Caller can
2211 * expects some charges will be back to res_counter later but cannot wait for
2212 * it.
2213 */
2214static void drain_all_stock_async(struct mem_cgroup *root_mem)
2215{
Michal Hockod38144b2011-07-26 16:08:28 -07002216 drain_all_stock(root_mem, false);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002217}
2218
2219/* This is a synchronous drain interface. */
Michal Hockod38144b2011-07-26 16:08:28 -07002220static void drain_all_stock_sync(struct mem_cgroup *root_mem)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002221{
2222 /* called when force_empty is called */
Michal Hockod38144b2011-07-26 16:08:28 -07002223 drain_all_stock(root_mem, true);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002224}
2225
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002226/*
2227 * This function drains percpu counter value from DEAD cpu and
2228 * move it to local cpu. Note that this function can be preempted.
2229 */
2230static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu)
2231{
2232 int i;
2233
2234 spin_lock(&mem->pcp_counter_lock);
2235 for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) {
Johannes Weiner7a159cc2011-03-23 16:42:38 -07002236 long x = per_cpu(mem->stat->count[i], cpu);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002237
2238 per_cpu(mem->stat->count[i], cpu) = 0;
2239 mem->nocpu_base.count[i] += x;
2240 }
Johannes Weinere9f89742011-03-23 16:42:37 -07002241 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
2242 unsigned long x = per_cpu(mem->stat->events[i], cpu);
2243
2244 per_cpu(mem->stat->events[i], cpu) = 0;
2245 mem->nocpu_base.events[i] += x;
2246 }
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07002247 /* need to clear ON_MOVE value, works as a kind of lock. */
2248 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
2249 spin_unlock(&mem->pcp_counter_lock);
2250}
2251
2252static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu)
2253{
2254 int idx = MEM_CGROUP_ON_MOVE;
2255
2256 spin_lock(&mem->pcp_counter_lock);
2257 per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx];
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002258 spin_unlock(&mem->pcp_counter_lock);
2259}
2260
2261static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002262 unsigned long action,
2263 void *hcpu)
2264{
2265 int cpu = (unsigned long)hcpu;
2266 struct memcg_stock_pcp *stock;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002267 struct mem_cgroup *iter;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002268
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07002269 if ((action == CPU_ONLINE)) {
2270 for_each_mem_cgroup_all(iter)
2271 synchronize_mem_cgroup_on_move(iter, cpu);
2272 return NOTIFY_OK;
2273 }
2274
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002275 if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002276 return NOTIFY_OK;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002277
2278 for_each_mem_cgroup_all(iter)
2279 mem_cgroup_drain_pcp_counter(iter, cpu);
2280
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002281 stock = &per_cpu(memcg_stock, cpu);
2282 drain_stock(stock);
2283 return NOTIFY_OK;
2284}
2285
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002286
2287/* See __mem_cgroup_try_charge() for details */
2288enum {
2289 CHARGE_OK, /* success */
2290 CHARGE_RETRY, /* need to retry but retry is not bad */
2291 CHARGE_NOMEM, /* we can't do more. return -ENOMEM */
2292 CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */
2293 CHARGE_OOM_DIE, /* the current is killed because of OOM */
2294};
2295
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002296static int mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
2297 unsigned int nr_pages, bool oom_check)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002298{
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002299 unsigned long csize = nr_pages * PAGE_SIZE;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002300 struct mem_cgroup *mem_over_limit;
2301 struct res_counter *fail_res;
2302 unsigned long flags = 0;
2303 int ret;
2304
2305 ret = res_counter_charge(&mem->res, csize, &fail_res);
2306
2307 if (likely(!ret)) {
2308 if (!do_swap_account)
2309 return CHARGE_OK;
2310 ret = res_counter_charge(&mem->memsw, csize, &fail_res);
2311 if (likely(!ret))
2312 return CHARGE_OK;
2313
KAMEZAWA Hiroyuki01c88e22011-01-25 15:07:27 -08002314 res_counter_uncharge(&mem->res, csize);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002315 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
2316 flags |= MEM_CGROUP_RECLAIM_NOSWAP;
2317 } else
2318 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
Johannes Weiner9221edb2011-02-01 15:52:42 -08002319 /*
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002320 * nr_pages can be either a huge page (HPAGE_PMD_NR), a batch
2321 * of regular pages (CHARGE_BATCH), or a single regular page (1).
Johannes Weiner9221edb2011-02-01 15:52:42 -08002322 *
2323 * Never reclaim on behalf of optional batching, retry with a
2324 * single page instead.
2325 */
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002326 if (nr_pages == CHARGE_BATCH)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002327 return CHARGE_RETRY;
2328
2329 if (!(gfp_mask & __GFP_WAIT))
2330 return CHARGE_WOULDBLOCK;
2331
2332 ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
Ying Han0ae5e892011-05-26 16:25:25 -07002333 gfp_mask, flags, NULL);
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002334 if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
Johannes Weiner19942822011-02-01 15:52:43 -08002335 return CHARGE_RETRY;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002336 /*
Johannes Weiner19942822011-02-01 15:52:43 -08002337 * Even though the limit is exceeded at this point, reclaim
2338 * may have been able to free some pages. Retry the charge
2339 * before killing the task.
2340 *
2341 * Only for regular pages, though: huge pages are rather
2342 * unlikely to succeed so close to the limit, and we fall back
2343 * to regular pages anyway in case of failure.
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002344 */
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002345 if (nr_pages == 1 && ret)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002346 return CHARGE_RETRY;
2347
2348 /*
2349 * At task move, charge accounts can be doubly counted. So, it's
2350 * better to wait until the end of task_move if something is going on.
2351 */
2352 if (mem_cgroup_wait_acct_move(mem_over_limit))
2353 return CHARGE_RETRY;
2354
2355 /* If we don't need to call oom-killer at el, return immediately */
2356 if (!oom_check)
2357 return CHARGE_NOMEM;
2358 /* check OOM */
2359 if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask))
2360 return CHARGE_OOM_DIE;
2361
2362 return CHARGE_RETRY;
2363}
2364
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002365/*
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002366 * Unlike exported interface, "oom" parameter is added. if oom==true,
2367 * oom-killer can be invoked.
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002368 */
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002369static int __mem_cgroup_try_charge(struct mm_struct *mm,
Andrea Arcangeliec168512011-01-13 15:46:56 -08002370 gfp_t gfp_mask,
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002371 unsigned int nr_pages,
2372 struct mem_cgroup **memcg,
2373 bool oom)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002374{
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002375 unsigned int batch = max(CHARGE_BATCH, nr_pages);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002376 int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2377 struct mem_cgroup *mem = NULL;
2378 int ret;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002379
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002380 /*
2381 * Unlike gloval-vm's OOM-kill, we're not in memory shortage
2382 * in system level. So, allow to go ahead dying process in addition to
2383 * MEMDIE process.
2384 */
2385 if (unlikely(test_thread_flag(TIF_MEMDIE)
2386 || fatal_signal_pending(current)))
2387 goto bypass;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002388
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002389 /*
Hugh Dickins3be91272008-02-07 00:14:19 -08002390 * We always charge the cgroup the mm_struct belongs to.
2391 * The mm_struct's mem_cgroup changes on task migration if the
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002392 * thread group leader migrates. It's possible that mm is not
2393 * set, if so charge the init_mm (happens for pagecache usage).
2394 */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002395 if (!*memcg && !mm)
2396 goto bypass;
2397again:
2398 if (*memcg) { /* css should be a valid one */
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002399 mem = *memcg;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002400 VM_BUG_ON(css_is_removed(&mem->css));
2401 if (mem_cgroup_is_root(mem))
2402 goto done;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002403 if (nr_pages == 1 && consume_stock(mem))
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002404 goto done;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002405 css_get(&mem->css);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002406 } else {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002407 struct task_struct *p;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002408
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002409 rcu_read_lock();
2410 p = rcu_dereference(mm->owner);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002411 /*
KAMEZAWA Hiroyukiebb76ce2010-12-29 14:07:11 -08002412 * Because we don't have task_lock(), "p" can exit.
2413 * In that case, "mem" can point to root or p can be NULL with
2414 * race with swapoff. Then, we have small risk of mis-accouning.
2415 * But such kind of mis-account by race always happens because
2416 * we don't have cgroup_mutex(). It's overkill and we allo that
2417 * small race, here.
2418 * (*) swapoff at el will charge against mm-struct not against
2419 * task-struct. So, mm->owner can be NULL.
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002420 */
2421 mem = mem_cgroup_from_task(p);
KAMEZAWA Hiroyukiebb76ce2010-12-29 14:07:11 -08002422 if (!mem || mem_cgroup_is_root(mem)) {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002423 rcu_read_unlock();
2424 goto done;
2425 }
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002426 if (nr_pages == 1 && consume_stock(mem)) {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002427 /*
2428 * It seems dagerous to access memcg without css_get().
2429 * But considering how consume_stok works, it's not
2430 * necessary. If consume_stock success, some charges
2431 * from this memcg are cached on this cpu. So, we
2432 * don't need to call css_get()/css_tryget() before
2433 * calling consume_stock().
2434 */
2435 rcu_read_unlock();
2436 goto done;
2437 }
2438 /* after here, we may be blocked. we need to get refcnt */
2439 if (!css_tryget(&mem->css)) {
2440 rcu_read_unlock();
2441 goto again;
2442 }
2443 rcu_read_unlock();
2444 }
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002445
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002446 do {
2447 bool oom_check;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002448
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002449 /* If killed, bypass charge */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002450 if (fatal_signal_pending(current)) {
2451 css_put(&mem->css);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002452 goto bypass;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002453 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002454
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002455 oom_check = false;
2456 if (oom && !nr_oom_retries) {
2457 oom_check = true;
2458 nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2459 }
Balbir Singh6d61ef42009-01-07 18:08:06 -08002460
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002461 ret = mem_cgroup_do_charge(mem, gfp_mask, batch, oom_check);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002462 switch (ret) {
2463 case CHARGE_OK:
2464 break;
2465 case CHARGE_RETRY: /* not in OOM situation but retry */
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002466 batch = nr_pages;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002467 css_put(&mem->css);
2468 mem = NULL;
2469 goto again;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002470 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002471 css_put(&mem->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002472 goto nomem;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002473 case CHARGE_NOMEM: /* OOM routine works */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002474 if (!oom) {
2475 css_put(&mem->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002476 goto nomem;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002477 }
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002478 /* If oom, we never return -ENOMEM */
2479 nr_oom_retries--;
2480 break;
2481 case CHARGE_OOM_DIE: /* Killed by OOM Killer */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002482 css_put(&mem->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002483 goto bypass;
Balbir Singh66e17072008-02-07 00:13:56 -08002484 }
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002485 } while (ret != CHARGE_OK);
2486
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002487 if (batch > nr_pages)
2488 refill_stock(mem, batch - nr_pages);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002489 css_put(&mem->css);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002490done:
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002491 *memcg = mem;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002492 return 0;
2493nomem:
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002494 *memcg = NULL;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002495 return -ENOMEM;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002496bypass:
2497 *memcg = NULL;
2498 return 0;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002499}
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002500
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002501/*
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002502 * Somemtimes we have to undo a charge we got by try_charge().
2503 * This function is for that and do uncharge, put css's refcnt.
2504 * gotten by try_charge().
2505 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002506static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
Johannes Weinere7018b82011-03-23 16:42:33 -07002507 unsigned int nr_pages)
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002508{
2509 if (!mem_cgroup_is_root(mem)) {
Johannes Weinere7018b82011-03-23 16:42:33 -07002510 unsigned long bytes = nr_pages * PAGE_SIZE;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002511
Johannes Weinere7018b82011-03-23 16:42:33 -07002512 res_counter_uncharge(&mem->res, bytes);
2513 if (do_swap_account)
2514 res_counter_uncharge(&mem->memsw, bytes);
2515 }
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002516}
2517
2518/*
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002519 * A helper function to get mem_cgroup from ID. must be called under
2520 * rcu_read_lock(). The caller must check css_is_removed() or some if
2521 * it's concern. (dropping refcnt from swap can be called against removed
2522 * memcg.)
2523 */
2524static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
2525{
2526 struct cgroup_subsys_state *css;
2527
2528 /* ID 0 is unused ID */
2529 if (!id)
2530 return NULL;
2531 css = css_lookup(&mem_cgroup_subsys, id);
2532 if (!css)
2533 return NULL;
2534 return container_of(css, struct mem_cgroup, css);
2535}
2536
Wu Fengguange42d9d52009-12-16 12:19:59 +01002537struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002538{
Wu Fengguange42d9d52009-12-16 12:19:59 +01002539 struct mem_cgroup *mem = NULL;
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002540 struct page_cgroup *pc;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002541 unsigned short id;
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002542 swp_entry_t ent;
2543
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002544 VM_BUG_ON(!PageLocked(page));
2545
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002546 pc = lookup_page_cgroup(page);
Daisuke Nishimurac0bd3f62009-04-30 15:08:11 -07002547 lock_page_cgroup(pc);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002548 if (PageCgroupUsed(pc)) {
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002549 mem = pc->mem_cgroup;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002550 if (mem && !css_tryget(&mem->css))
2551 mem = NULL;
Wu Fengguange42d9d52009-12-16 12:19:59 +01002552 } else if (PageSwapCache(page)) {
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002553 ent.val = page_private(page);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002554 id = lookup_swap_cgroup(ent);
2555 rcu_read_lock();
2556 mem = mem_cgroup_lookup(id);
2557 if (mem && !css_tryget(&mem->css))
2558 mem = NULL;
2559 rcu_read_unlock();
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002560 }
Daisuke Nishimurac0bd3f62009-04-30 15:08:11 -07002561 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002562 return mem;
2563}
2564
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002565static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
Johannes Weiner5564e882011-03-23 16:42:29 -07002566 struct page *page,
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002567 unsigned int nr_pages,
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002568 struct page_cgroup *pc,
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002569 enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002570{
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002571 lock_page_cgroup(pc);
2572 if (unlikely(PageCgroupUsed(pc))) {
2573 unlock_page_cgroup(pc);
Johannes Weinere7018b82011-03-23 16:42:33 -07002574 __mem_cgroup_cancel_charge(mem, nr_pages);
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002575 return;
2576 }
2577 /*
2578 * we don't need page_cgroup_lock about tail pages, becase they are not
2579 * accessed by any other context at this point.
2580 */
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002581 pc->mem_cgroup = mem;
KAMEZAWA Hiroyuki261fb612009-09-23 15:56:33 -07002582 /*
2583 * We access a page_cgroup asynchronously without lock_page_cgroup().
2584 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
2585 * is accessed after testing USED bit. To make pc->mem_cgroup visible
2586 * before USED bit, we need memory barrier here.
2587 * See mem_cgroup_add_lru_list(), etc.
2588 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002589 smp_wmb();
Balbir Singh4b3bde42009-09-23 15:56:32 -07002590 switch (ctype) {
2591 case MEM_CGROUP_CHARGE_TYPE_CACHE:
2592 case MEM_CGROUP_CHARGE_TYPE_SHMEM:
2593 SetPageCgroupCache(pc);
2594 SetPageCgroupUsed(pc);
2595 break;
2596 case MEM_CGROUP_CHARGE_TYPE_MAPPED:
2597 ClearPageCgroupCache(pc);
2598 SetPageCgroupUsed(pc);
2599 break;
2600 default:
2601 break;
2602 }
Hugh Dickins3be91272008-02-07 00:14:19 -08002603
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002604 mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), nr_pages);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002605 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki430e48632010-03-10 15:22:30 -08002606 /*
2607 * "charge_statistics" updated event counter. Then, check it.
2608 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
2609 * if they exceeds softlimit.
2610 */
Johannes Weiner5564e882011-03-23 16:42:29 -07002611 memcg_check_events(mem, page);
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002612}
2613
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002614#ifdef CONFIG_TRANSPARENT_HUGEPAGE
2615
2616#define PCGF_NOCOPY_AT_SPLIT ((1 << PCG_LOCK) | (1 << PCG_MOVE_LOCK) |\
2617 (1 << PCG_ACCT_LRU) | (1 << PCG_MIGRATION))
2618/*
2619 * Because tail pages are not marked as "used", set it. We're under
2620 * zone->lru_lock, 'splitting on pmd' and compund_lock.
2621 */
2622void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail)
2623{
2624 struct page_cgroup *head_pc = lookup_page_cgroup(head);
2625 struct page_cgroup *tail_pc = lookup_page_cgroup(tail);
2626 unsigned long flags;
2627
KAMEZAWA Hiroyuki3d37c4a2011-01-25 15:07:28 -08002628 if (mem_cgroup_disabled())
2629 return;
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002630 /*
KAMEZAWA Hiroyukiece35ca2011-01-20 14:44:24 -08002631 * We have no races with charge/uncharge but will have races with
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002632 * page state accounting.
2633 */
2634 move_lock_page_cgroup(head_pc, &flags);
2635
2636 tail_pc->mem_cgroup = head_pc->mem_cgroup;
2637 smp_wmb(); /* see __commit_charge() */
KAMEZAWA Hiroyukiece35ca2011-01-20 14:44:24 -08002638 if (PageCgroupAcctLRU(head_pc)) {
2639 enum lru_list lru;
2640 struct mem_cgroup_per_zone *mz;
2641
2642 /*
2643 * LRU flags cannot be copied because we need to add tail
2644 *.page to LRU by generic call and our hook will be called.
2645 * We hold lru_lock, then, reduce counter directly.
2646 */
2647 lru = page_lru(head);
Johannes Weiner97a6c372011-03-23 16:42:27 -07002648 mz = page_cgroup_zoneinfo(head_pc->mem_cgroup, head);
KAMEZAWA Hiroyukiece35ca2011-01-20 14:44:24 -08002649 MEM_CGROUP_ZSTAT(mz, lru) -= 1;
2650 }
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002651 tail_pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
2652 move_unlock_page_cgroup(head_pc, &flags);
2653}
2654#endif
2655
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002656/**
Johannes Weinerde3638d2011-03-23 16:42:28 -07002657 * mem_cgroup_move_account - move account of the page
Johannes Weiner5564e882011-03-23 16:42:29 -07002658 * @page: the page
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002659 * @nr_pages: number of regular pages (>1 for huge pages)
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002660 * @pc: page_cgroup of the page.
2661 * @from: mem_cgroup which the page is moved from.
2662 * @to: mem_cgroup which the page is moved to. @from != @to.
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002663 * @uncharge: whether we should call uncharge and css_put against @from.
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002664 *
2665 * The caller must confirm following.
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002666 * - page is not on LRU (isolate_page() is useful.)
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002667 * - compound_lock is held when nr_pages > 1
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002668 *
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002669 * This function doesn't do "charge" nor css_get to new cgroup. It should be
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002670 * done by a caller(__mem_cgroup_try_charge would be useful). If @uncharge is
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002671 * true, this function does "uncharge" from old cgroup, but it doesn't if
2672 * @uncharge is false, so a caller should do "uncharge".
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002673 */
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002674static int mem_cgroup_move_account(struct page *page,
2675 unsigned int nr_pages,
2676 struct page_cgroup *pc,
2677 struct mem_cgroup *from,
2678 struct mem_cgroup *to,
2679 bool uncharge)
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002680{
Johannes Weinerde3638d2011-03-23 16:42:28 -07002681 unsigned long flags;
2682 int ret;
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08002683
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002684 VM_BUG_ON(from == to);
Johannes Weiner5564e882011-03-23 16:42:29 -07002685 VM_BUG_ON(PageLRU(page));
Johannes Weinerde3638d2011-03-23 16:42:28 -07002686 /*
2687 * The page is isolated from LRU. So, collapse function
2688 * will not handle this page. But page splitting can happen.
2689 * Do this check under compound_page_lock(). The caller should
2690 * hold it.
2691 */
2692 ret = -EBUSY;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002693 if (nr_pages > 1 && !PageTransHuge(page))
Johannes Weinerde3638d2011-03-23 16:42:28 -07002694 goto out;
2695
2696 lock_page_cgroup(pc);
2697
2698 ret = -EINVAL;
2699 if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
2700 goto unlock;
2701
2702 move_lock_page_cgroup(pc, &flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002703
KAMEZAWA Hiroyuki8725d542010-04-06 14:35:05 -07002704 if (PageCgroupFileMapped(pc)) {
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08002705 /* Update mapped_file data for mem_cgroup */
2706 preempt_disable();
2707 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
2708 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
2709 preempt_enable();
Balbir Singhd69b0422009-06-17 16:26:34 -07002710 }
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08002711 mem_cgroup_charge_statistics(from, PageCgroupCache(pc), -nr_pages);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002712 if (uncharge)
2713 /* This is not "cancel", but cancel_charge does all we need. */
Johannes Weinere7018b82011-03-23 16:42:33 -07002714 __mem_cgroup_cancel_charge(from, nr_pages);
Balbir Singhd69b0422009-06-17 16:26:34 -07002715
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002716 /* caller should have done css_get */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002717 pc->mem_cgroup = to;
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08002718 mem_cgroup_charge_statistics(to, PageCgroupCache(pc), nr_pages);
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002719 /*
2720 * We charges against "to" which may not have any tasks. Then, "to"
2721 * can be under rmdir(). But in current implementation, caller of
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08002722 * this function is just force_empty() and move charge, so it's
Lucas De Marchi25985ed2011-03-30 22:57:33 -03002723 * guaranteed that "to" is never removed. So, we don't check rmdir
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08002724 * status here.
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002725 */
Johannes Weinerde3638d2011-03-23 16:42:28 -07002726 move_unlock_page_cgroup(pc, &flags);
2727 ret = 0;
2728unlock:
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08002729 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08002730 /*
2731 * check events
2732 */
Johannes Weiner5564e882011-03-23 16:42:29 -07002733 memcg_check_events(to, page);
2734 memcg_check_events(from, page);
Johannes Weinerde3638d2011-03-23 16:42:28 -07002735out:
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002736 return ret;
2737}
2738
2739/*
2740 * move charges to its parent.
2741 */
2742
Johannes Weiner5564e882011-03-23 16:42:29 -07002743static int mem_cgroup_move_parent(struct page *page,
2744 struct page_cgroup *pc,
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002745 struct mem_cgroup *child,
2746 gfp_t gfp_mask)
2747{
2748 struct cgroup *cg = child->css.cgroup;
2749 struct cgroup *pcg = cg->parent;
2750 struct mem_cgroup *parent;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002751 unsigned int nr_pages;
Andrew Morton4be44892011-03-23 16:42:39 -07002752 unsigned long uninitialized_var(flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002753 int ret;
2754
2755 /* Is ROOT ? */
2756 if (!pcg)
2757 return -EINVAL;
2758
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08002759 ret = -EBUSY;
2760 if (!get_page_unless_zero(page))
2761 goto out;
2762 if (isolate_lru_page(page))
2763 goto put;
KAMEZAWA Hiroyuki52dbb902011-01-25 15:07:29 -08002764
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002765 nr_pages = hpage_nr_pages(page);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002766
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002767 parent = mem_cgroup_from_cont(pcg);
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002768 ret = __mem_cgroup_try_charge(NULL, gfp_mask, nr_pages, &parent, false);
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002769 if (ret || !parent)
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08002770 goto put_back;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002771
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002772 if (nr_pages > 1)
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08002773 flags = compound_lock_irqsave(page);
2774
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002775 ret = mem_cgroup_move_account(page, nr_pages, pc, child, parent, true);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002776 if (ret)
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002777 __mem_cgroup_cancel_charge(parent, nr_pages);
Jesper Juhl8dba4742011-01-25 15:07:24 -08002778
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002779 if (nr_pages > 1)
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08002780 compound_unlock_irqrestore(page, flags);
Jesper Juhl8dba4742011-01-25 15:07:24 -08002781put_back:
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002782 putback_lru_page(page);
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08002783put:
Daisuke Nishimura40d58132009-01-15 13:51:12 -08002784 put_page(page);
Daisuke Nishimura57f9fd72009-12-15 16:47:11 -08002785out:
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002786 return ret;
2787}
2788
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002789/*
2790 * Charge the memory controller for page usage.
2791 * Return
2792 * 0 if the charge was successful
2793 * < 0 if the cgroup is over its limit
2794 */
2795static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002796 gfp_t gfp_mask, enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002797{
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002798 struct mem_cgroup *mem = NULL;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002799 unsigned int nr_pages = 1;
Johannes Weiner8493ae42011-02-01 15:52:44 -08002800 struct page_cgroup *pc;
2801 bool oom = true;
2802 int ret;
Andrea Arcangeliec168512011-01-13 15:46:56 -08002803
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08002804 if (PageTransHuge(page)) {
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002805 nr_pages <<= compound_order(page);
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08002806 VM_BUG_ON(!PageTransHuge(page));
Johannes Weiner8493ae42011-02-01 15:52:44 -08002807 /*
2808 * Never OOM-kill a process for a huge page. The
2809 * fault handler will fall back to regular pages.
2810 */
2811 oom = false;
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08002812 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002813
2814 pc = lookup_page_cgroup(page);
Johannes Weineraf4a6622011-03-23 16:42:24 -07002815 BUG_ON(!pc); /* XXX: remove this and move pc lookup into commit */
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002816
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002817 ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &mem, oom);
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002818 if (ret || !mem)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002819 return ret;
2820
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002821 __mem_cgroup_commit_charge(mem, page, nr_pages, pc, ctype);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002822 return 0;
2823}
2824
2825int mem_cgroup_newpage_charge(struct page *page,
2826 struct mm_struct *mm, gfp_t gfp_mask)
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08002827{
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08002828 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07002829 return 0;
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07002830 /*
2831 * If already mapped, we don't have to account.
2832 * If page cache, page->mapping has address_space.
2833 * But page->mapping may have out-of-use anon_vma pointer,
2834 * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
2835 * is NULL.
2836 */
2837 if (page_mapped(page) || (page->mapping && !PageAnon(page)))
2838 return 0;
2839 if (unlikely(!mm))
2840 mm = &init_mm;
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08002841 return mem_cgroup_charge_common(page, mm, gfp_mask,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002842 MEM_CGROUP_CHARGE_TYPE_MAPPED);
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08002843}
2844
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002845static void
2846__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2847 enum charge_type ctype);
2848
KAMEZAWA Hiroyuki5a6475a2011-03-23 16:42:42 -07002849static void
2850__mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *mem,
2851 enum charge_type ctype)
2852{
2853 struct page_cgroup *pc = lookup_page_cgroup(page);
2854 /*
2855 * In some case, SwapCache, FUSE(splice_buf->radixtree), the page
2856 * is already on LRU. It means the page may on some other page_cgroup's
2857 * LRU. Take care of it.
2858 */
2859 mem_cgroup_lru_del_before_commit(page);
2860 __mem_cgroup_commit_charge(mem, page, 1, pc, ctype);
2861 mem_cgroup_lru_add_after_commit(page);
2862 return;
2863}
2864
Balbir Singhe1a1cd52008-02-07 00:14:02 -08002865int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
2866 gfp_t gfp_mask)
Balbir Singh8697d332008-02-07 00:13:59 -08002867{
KAMEZAWA Hiroyuki5a6475a2011-03-23 16:42:42 -07002868 struct mem_cgroup *mem = NULL;
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002869 int ret;
2870
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08002871 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07002872 return 0;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002873 if (PageCompound(page))
2874 return 0;
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002875
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002876 if (unlikely(!mm))
Balbir Singh8697d332008-02-07 00:13:59 -08002877 mm = &init_mm;
KAMEZAWA Hiroyukiaccf1632008-07-25 01:47:17 -07002878
KAMEZAWA Hiroyuki5a6475a2011-03-23 16:42:42 -07002879 if (page_is_file_cache(page)) {
2880 ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, &mem, true);
2881 if (ret || !mem)
2882 return ret;
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002883
KAMEZAWA Hiroyuki5a6475a2011-03-23 16:42:42 -07002884 /*
2885 * FUSE reuses pages without going through the final
2886 * put that would remove them from the LRU list, make
2887 * sure that they get relinked properly.
2888 */
2889 __mem_cgroup_commit_charge_lrucare(page, mem,
2890 MEM_CGROUP_CHARGE_TYPE_CACHE);
2891 return ret;
2892 }
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002893 /* shmem */
2894 if (PageSwapCache(page)) {
2895 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2896 if (!ret)
2897 __mem_cgroup_commit_charge_swapin(page, mem,
2898 MEM_CGROUP_CHARGE_TYPE_SHMEM);
2899 } else
2900 ret = mem_cgroup_charge_common(page, mm, gfp_mask,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07002901 MEM_CGROUP_CHARGE_TYPE_SHMEM);
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002902
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002903 return ret;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07002904}
2905
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002906/*
2907 * While swap-in, try_charge -> commit or cancel, the page is locked.
2908 * And when try_charge() successfully returns, one refcnt to memcg without
Uwe Kleine-König21ae2952009-10-07 15:21:09 +02002909 * struct page_cgroup is acquired. This refcnt will be consumed by
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002910 * "commit()" or removed by "cancel()"
2911 */
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002912int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
2913 struct page *page,
2914 gfp_t mask, struct mem_cgroup **ptr)
2915{
2916 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002917 int ret;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002918
KAMEZAWA Hiroyuki56039ef2011-03-23 16:42:19 -07002919 *ptr = NULL;
2920
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08002921 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002922 return 0;
2923
2924 if (!do_swap_account)
2925 goto charge_cur_mm;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002926 /*
2927 * A racing thread's fault, or swapoff, may have already updated
Hugh Dickins407f9c82009-12-14 17:59:30 -08002928 * the pte, and even removed page from swap cache: in those cases
2929 * do_swap_page()'s pte_same() test will fail; but there's also a
2930 * KSM case which does need to charge the page.
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002931 */
2932 if (!PageSwapCache(page))
Hugh Dickins407f9c82009-12-14 17:59:30 -08002933 goto charge_cur_mm;
Wu Fengguange42d9d52009-12-16 12:19:59 +01002934 mem = try_get_mem_cgroup_from_page(page);
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002935 if (!mem)
2936 goto charge_cur_mm;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002937 *ptr = mem;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002938 ret = __mem_cgroup_try_charge(NULL, mask, 1, ptr, true);
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002939 css_put(&mem->css);
2940 return ret;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002941charge_cur_mm:
2942 if (unlikely(!mm))
2943 mm = &init_mm;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002944 return __mem_cgroup_try_charge(mm, mask, 1, ptr, true);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002945}
2946
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002947static void
2948__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2949 enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002950{
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08002951 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002952 return;
2953 if (!ptr)
2954 return;
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002955 cgroup_exclude_rmdir(&ptr->css);
KAMEZAWA Hiroyuki5a6475a2011-03-23 16:42:42 -07002956
2957 __mem_cgroup_commit_charge_lrucare(page, ptr, ctype);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002958 /*
2959 * Now swap is on-memory. This means this page may be
2960 * counted both as mem and swap....double count.
KAMEZAWA Hiroyuki03f3c432009-01-07 18:08:31 -08002961 * Fix it by uncharging from memsw. Basically, this SwapCache is stable
2962 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
2963 * may call delete_from_swap_cache() before reach here.
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002964 */
KAMEZAWA Hiroyuki03f3c432009-01-07 18:08:31 -08002965 if (do_swap_account && PageSwapCache(page)) {
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002966 swp_entry_t ent = {.val = page_private(page)};
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002967 unsigned short id;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002968 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002969
2970 id = swap_cgroup_record(ent, 0);
2971 rcu_read_lock();
2972 memcg = mem_cgroup_lookup(id);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002973 if (memcg) {
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002974 /*
2975 * This recorded memcg can be obsolete one. So, avoid
2976 * calling css_tryget
2977 */
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002978 if (!mem_cgroup_is_root(memcg))
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -07002979 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002980 mem_cgroup_swap_statistics(memcg, false);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002981 mem_cgroup_put(memcg);
2982 }
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002983 rcu_read_unlock();
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08002984 }
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07002985 /*
2986 * At swapin, we may charge account against cgroup which has no tasks.
2987 * So, rmdir()->pre_destroy() can be called while we do this charge.
2988 * In that case, we need to call pre_destroy() again. check it here.
2989 */
2990 cgroup_release_and_wakeup_rmdir(&ptr->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002991}
2992
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07002993void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
2994{
2995 __mem_cgroup_commit_charge_swapin(page, ptr,
2996 MEM_CGROUP_CHARGE_TYPE_MAPPED);
2997}
2998
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002999void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
3000{
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08003001 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003002 return;
3003 if (!mem)
3004 return;
Johannes Weinere7018b82011-03-23 16:42:33 -07003005 __mem_cgroup_cancel_charge(mem, 1);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003006}
3007
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003008static void mem_cgroup_do_uncharge(struct mem_cgroup *mem,
3009 unsigned int nr_pages,
3010 const enum charge_type ctype)
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003011{
3012 struct memcg_batch_info *batch = NULL;
3013 bool uncharge_memsw = true;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003014
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003015 /* If swapout, usage of swap doesn't decrease */
3016 if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
3017 uncharge_memsw = false;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003018
3019 batch = &current->memcg_batch;
3020 /*
3021 * In usual, we do css_get() when we remember memcg pointer.
3022 * But in this case, we keep res->usage until end of a series of
3023 * uncharges. Then, it's ok to ignore memcg's refcnt.
3024 */
3025 if (!batch->memcg)
3026 batch->memcg = mem;
3027 /*
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003028 * do_batch > 0 when unmapping pages or inode invalidate/truncate.
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003029 * In those cases, all pages freed continuously can be expected to be in
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003030 * the same cgroup and we have chance to coalesce uncharges.
3031 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
3032 * because we want to do uncharge as soon as possible.
3033 */
3034
3035 if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
3036 goto direct_uncharge;
3037
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003038 if (nr_pages > 1)
Andrea Arcangeliec168512011-01-13 15:46:56 -08003039 goto direct_uncharge;
3040
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003041 /*
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003042 * In typical case, batch->memcg == mem. This means we can
3043 * merge a series of uncharges to an uncharge of res_counter.
3044 * If not, we uncharge res_counter ony by one.
3045 */
3046 if (batch->memcg != mem)
3047 goto direct_uncharge;
3048 /* remember freed charge and uncharge it later */
Johannes Weiner7ffd4ca2011-03-23 16:42:35 -07003049 batch->nr_pages++;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003050 if (uncharge_memsw)
Johannes Weiner7ffd4ca2011-03-23 16:42:35 -07003051 batch->memsw_nr_pages++;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003052 return;
3053direct_uncharge:
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003054 res_counter_uncharge(&mem->res, nr_pages * PAGE_SIZE);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003055 if (uncharge_memsw)
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003056 res_counter_uncharge(&mem->memsw, nr_pages * PAGE_SIZE);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003057 if (unlikely(batch->memcg != mem))
3058 memcg_oom_recover(mem);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003059 return;
3060}
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003061
Balbir Singh8697d332008-02-07 00:13:59 -08003062/*
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07003063 * uncharge if !page_mapped(page)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08003064 */
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003065static struct mem_cgroup *
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07003066__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08003067{
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003068 struct mem_cgroup *mem = NULL;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003069 unsigned int nr_pages = 1;
3070 struct page_cgroup *pc;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08003071
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08003072 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003073 return NULL;
Balbir Singh40779602008-04-04 14:29:59 -07003074
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003075 if (PageSwapCache(page))
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003076 return NULL;
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003077
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08003078 if (PageTransHuge(page)) {
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003079 nr_pages <<= compound_order(page);
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08003080 VM_BUG_ON(!PageTransHuge(page));
3081 }
Balbir Singh8697d332008-02-07 00:13:59 -08003082 /*
Balbir Singh3c541e12008-02-07 00:14:41 -08003083 * Check if our page_cgroup is valid
Balbir Singh8697d332008-02-07 00:13:59 -08003084 */
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003085 pc = lookup_page_cgroup(page);
3086 if (unlikely(!pc || !PageCgroupUsed(pc)))
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003087 return NULL;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08003088
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003089 lock_page_cgroup(pc);
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003090
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003091 mem = pc->mem_cgroup;
3092
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003093 if (!PageCgroupUsed(pc))
3094 goto unlock_out;
3095
3096 switch (ctype) {
3097 case MEM_CGROUP_CHARGE_TYPE_MAPPED:
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07003098 case MEM_CGROUP_CHARGE_TYPE_DROP:
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003099 /* See mem_cgroup_prepare_migration() */
3100 if (page_mapped(page) || PageCgroupMigration(pc))
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003101 goto unlock_out;
3102 break;
3103 case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
3104 if (!PageAnon(page)) { /* Shared memory */
3105 if (page->mapping && !page_is_file_cache(page))
3106 goto unlock_out;
3107 } else if (page_mapped(page)) /* Anon */
3108 goto unlock_out;
3109 break;
3110 default:
3111 break;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003112 }
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003113
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003114 mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), -nr_pages);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07003115
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003116 ClearPageCgroupUsed(pc);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08003117 /*
3118 * pc->mem_cgroup is not cleared here. It will be accessed when it's
3119 * freed from LRU. This is safe because uncharged page is expected not
3120 * to be reused (freed soon). Exception is SwapCache, it's handled by
3121 * special functions.
3122 */
Hugh Dickinsb9c565d2008-03-04 14:29:11 -08003123
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003124 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07003125 /*
3126 * even after unlock, we have mem->res.usage here and this memcg
3127 * will never be freed.
3128 */
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08003129 memcg_check_events(mem, page);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07003130 if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
3131 mem_cgroup_swap_statistics(mem, true);
3132 mem_cgroup_get(mem);
3133 }
3134 if (!mem_cgroup_is_root(mem))
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003135 mem_cgroup_do_uncharge(mem, nr_pages, ctype);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08003136
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003137 return mem;
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003138
3139unlock_out:
3140 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003141 return NULL;
Balbir Singh3c541e12008-02-07 00:14:41 -08003142}
3143
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07003144void mem_cgroup_uncharge_page(struct page *page)
3145{
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003146 /* early check. */
3147 if (page_mapped(page))
3148 return;
3149 if (page->mapping && !PageAnon(page))
3150 return;
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07003151 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
3152}
3153
3154void mem_cgroup_uncharge_cache_page(struct page *page)
3155{
3156 VM_BUG_ON(page_mapped(page));
KAMEZAWA Hiroyukib7abea92008-10-18 20:28:09 -07003157 VM_BUG_ON(page->mapping);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07003158 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
3159}
3160
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003161/*
3162 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
3163 * In that cases, pages are freed continuously and we can expect pages
3164 * are in the same memcg. All these calls itself limits the number of
3165 * pages freed at once, then uncharge_start/end() is called properly.
3166 * This may be called prural(2) times in a context,
3167 */
3168
3169void mem_cgroup_uncharge_start(void)
3170{
3171 current->memcg_batch.do_batch++;
3172 /* We can do nest. */
3173 if (current->memcg_batch.do_batch == 1) {
3174 current->memcg_batch.memcg = NULL;
Johannes Weiner7ffd4ca2011-03-23 16:42:35 -07003175 current->memcg_batch.nr_pages = 0;
3176 current->memcg_batch.memsw_nr_pages = 0;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003177 }
3178}
3179
3180void mem_cgroup_uncharge_end(void)
3181{
3182 struct memcg_batch_info *batch = &current->memcg_batch;
3183
3184 if (!batch->do_batch)
3185 return;
3186
3187 batch->do_batch--;
3188 if (batch->do_batch) /* If stacked, do nothing. */
3189 return;
3190
3191 if (!batch->memcg)
3192 return;
3193 /*
3194 * This "batch->memcg" is valid without any css_get/put etc...
3195 * bacause we hide charges behind us.
3196 */
Johannes Weiner7ffd4ca2011-03-23 16:42:35 -07003197 if (batch->nr_pages)
3198 res_counter_uncharge(&batch->memcg->res,
3199 batch->nr_pages * PAGE_SIZE);
3200 if (batch->memsw_nr_pages)
3201 res_counter_uncharge(&batch->memcg->memsw,
3202 batch->memsw_nr_pages * PAGE_SIZE);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003203 memcg_oom_recover(batch->memcg);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003204 /* forget this pointer (for sanity check) */
3205 batch->memcg = NULL;
3206}
3207
Daisuke Nishimurae767e052009-05-28 14:34:28 -07003208#ifdef CONFIG_SWAP
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003209/*
Daisuke Nishimurae767e052009-05-28 14:34:28 -07003210 * called after __delete_from_swap_cache() and drop "page" account.
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003211 * memcg information is recorded to swap_cgroup of "ent"
3212 */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07003213void
3214mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003215{
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003216 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07003217 int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003218
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07003219 if (!swapout) /* this was a swap cache but the swap is unused ! */
3220 ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
3221
3222 memcg = __mem_cgroup_uncharge_common(page, ctype);
3223
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07003224 /*
3225 * record memcg information, if swapout && memcg != NULL,
3226 * mem_cgroup_get() was called in uncharge().
3227 */
3228 if (do_swap_account && swapout && memcg)
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07003229 swap_cgroup_record(ent, css_id(&memcg->css));
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003230}
Daisuke Nishimurae767e052009-05-28 14:34:28 -07003231#endif
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003232
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003233#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
3234/*
3235 * called from swap_entry_free(). remove record in swap_cgroup and
3236 * uncharge "memsw" account.
3237 */
3238void mem_cgroup_uncharge_swap(swp_entry_t ent)
3239{
3240 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07003241 unsigned short id;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003242
3243 if (!do_swap_account)
3244 return;
3245
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07003246 id = swap_cgroup_record(ent, 0);
3247 rcu_read_lock();
3248 memcg = mem_cgroup_lookup(id);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003249 if (memcg) {
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07003250 /*
3251 * We uncharge this because swap is freed.
3252 * This memcg can be obsolete one. We avoid calling css_tryget
3253 */
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003254 if (!mem_cgroup_is_root(memcg))
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -07003255 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003256 mem_cgroup_swap_statistics(memcg, false);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003257 mem_cgroup_put(memcg);
3258 }
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07003259 rcu_read_unlock();
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003260}
Daisuke Nishimura02491442010-03-10 15:22:17 -08003261
3262/**
3263 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
3264 * @entry: swap entry to be moved
3265 * @from: mem_cgroup which the entry is moved from
3266 * @to: mem_cgroup which the entry is moved to
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08003267 * @need_fixup: whether we should fixup res_counters and refcounts.
Daisuke Nishimura02491442010-03-10 15:22:17 -08003268 *
3269 * It succeeds only when the swap_cgroup's record for this entry is the same
3270 * as the mem_cgroup's id of @from.
3271 *
3272 * Returns 0 on success, -EINVAL on failure.
3273 *
3274 * The caller must have charged to @to, IOW, called res_counter_charge() about
3275 * both res and memsw, and called css_get().
3276 */
3277static int mem_cgroup_move_swap_account(swp_entry_t entry,
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08003278 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
Daisuke Nishimura02491442010-03-10 15:22:17 -08003279{
3280 unsigned short old_id, new_id;
3281
3282 old_id = css_id(&from->css);
3283 new_id = css_id(&to->css);
3284
3285 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08003286 mem_cgroup_swap_statistics(from, false);
Daisuke Nishimura02491442010-03-10 15:22:17 -08003287 mem_cgroup_swap_statistics(to, true);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08003288 /*
3289 * This function is only called from task migration context now.
3290 * It postpones res_counter and refcount handling till the end
3291 * of task migration(mem_cgroup_clear_mc()) for performance
3292 * improvement. But we cannot postpone mem_cgroup_get(to)
3293 * because if the process that has been moved to @to does
3294 * swap-in, the refcount of @to might be decreased to 0.
3295 */
Daisuke Nishimura02491442010-03-10 15:22:17 -08003296 mem_cgroup_get(to);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08003297 if (need_fixup) {
3298 if (!mem_cgroup_is_root(from))
3299 res_counter_uncharge(&from->memsw, PAGE_SIZE);
3300 mem_cgroup_put(from);
3301 /*
3302 * we charged both to->res and to->memsw, so we should
3303 * uncharge to->res.
3304 */
3305 if (!mem_cgroup_is_root(to))
3306 res_counter_uncharge(&to->res, PAGE_SIZE);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08003307 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08003308 return 0;
3309 }
3310 return -EINVAL;
3311}
3312#else
3313static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08003314 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
Daisuke Nishimura02491442010-03-10 15:22:17 -08003315{
3316 return -EINVAL;
3317}
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003318#endif
3319
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08003320/*
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08003321 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
3322 * page belongs to.
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08003323 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003324int mem_cgroup_prepare_migration(struct page *page,
Miklos Szeredief6a3c62011-03-22 16:30:52 -07003325 struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask)
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08003326{
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07003327 struct mem_cgroup *mem = NULL;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003328 struct page_cgroup *pc;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003329 enum charge_type ctype;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07003330 int ret = 0;
Hugh Dickins8869b8f2008-03-04 14:29:09 -08003331
KAMEZAWA Hiroyuki56039ef2011-03-23 16:42:19 -07003332 *ptr = NULL;
3333
Andrea Arcangeliec168512011-01-13 15:46:56 -08003334 VM_BUG_ON(PageTransHuge(page));
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08003335 if (mem_cgroup_disabled())
Balbir Singh40779602008-04-04 14:29:59 -07003336 return 0;
3337
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003338 pc = lookup_page_cgroup(page);
3339 lock_page_cgroup(pc);
3340 if (PageCgroupUsed(pc)) {
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07003341 mem = pc->mem_cgroup;
3342 css_get(&mem->css);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003343 /*
3344 * At migrating an anonymous page, its mapcount goes down
3345 * to 0 and uncharge() will be called. But, even if it's fully
3346 * unmapped, migration may fail and this page has to be
3347 * charged again. We set MIGRATION flag here and delay uncharge
3348 * until end_migration() is called
3349 *
3350 * Corner Case Thinking
3351 * A)
3352 * When the old page was mapped as Anon and it's unmap-and-freed
3353 * while migration was ongoing.
3354 * If unmap finds the old page, uncharge() of it will be delayed
3355 * until end_migration(). If unmap finds a new page, it's
3356 * uncharged when it make mapcount to be 1->0. If unmap code
3357 * finds swap_migration_entry, the new page will not be mapped
3358 * and end_migration() will find it(mapcount==0).
3359 *
3360 * B)
3361 * When the old page was mapped but migraion fails, the kernel
3362 * remaps it. A charge for it is kept by MIGRATION flag even
3363 * if mapcount goes down to 0. We can do remap successfully
3364 * without charging it again.
3365 *
3366 * C)
3367 * The "old" page is under lock_page() until the end of
3368 * migration, so, the old page itself will not be swapped-out.
3369 * If the new page is swapped out before end_migraton, our
3370 * hook to usual swap-out path will catch the event.
3371 */
3372 if (PageAnon(page))
3373 SetPageCgroupMigration(pc);
Hugh Dickinsb9c565d2008-03-04 14:29:11 -08003374 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003375 unlock_page_cgroup(pc);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003376 /*
3377 * If the page is not charged at this point,
3378 * we return here.
3379 */
3380 if (!mem)
3381 return 0;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08003382
Andrea Arcangeli93d5c9b2010-04-23 13:17:39 -04003383 *ptr = mem;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003384 ret = __mem_cgroup_try_charge(NULL, gfp_mask, 1, ptr, false);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003385 css_put(&mem->css);/* drop extra refcnt */
3386 if (ret || *ptr == NULL) {
3387 if (PageAnon(page)) {
3388 lock_page_cgroup(pc);
3389 ClearPageCgroupMigration(pc);
3390 unlock_page_cgroup(pc);
3391 /*
3392 * The old page may be fully unmapped while we kept it.
3393 */
3394 mem_cgroup_uncharge_page(page);
3395 }
3396 return -ENOMEM;
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07003397 }
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003398 /*
3399 * We charge new page before it's used/mapped. So, even if unlock_page()
3400 * is called before end_migration, we can catch all events on this new
3401 * page. In the case new page is migrated but not remapped, new page's
3402 * mapcount will be finally 0 and we call uncharge in end_migration().
3403 */
3404 pc = lookup_page_cgroup(newpage);
3405 if (PageAnon(page))
3406 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
3407 else if (page_is_file_cache(page))
3408 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
3409 else
3410 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003411 __mem_cgroup_commit_charge(mem, page, 1, pc, ctype);
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07003412 return ret;
3413}
Hugh Dickinsfb59e9f2008-03-04 14:29:16 -08003414
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07003415/* remove redundant charge if migration failed*/
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08003416void mem_cgroup_end_migration(struct mem_cgroup *mem,
Daisuke Nishimura50de1dd2011-01-13 15:47:43 -08003417 struct page *oldpage, struct page *newpage, bool migration_ok)
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07003418{
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003419 struct page *used, *unused;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08003420 struct page_cgroup *pc;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08003421
3422 if (!mem)
3423 return;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003424 /* blocks rmdir() */
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07003425 cgroup_exclude_rmdir(&mem->css);
Daisuke Nishimura50de1dd2011-01-13 15:47:43 -08003426 if (!migration_ok) {
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003427 used = oldpage;
3428 unused = newpage;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08003429 } else {
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003430 used = newpage;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08003431 unused = oldpage;
3432 }
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07003433 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003434 * We disallowed uncharge of pages under migration because mapcount
3435 * of the page goes down to zero, temporarly.
3436 * Clear the flag and check the page should be charged.
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07003437 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003438 pc = lookup_page_cgroup(oldpage);
3439 lock_page_cgroup(pc);
3440 ClearPageCgroupMigration(pc);
3441 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08003442
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003443 __mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE);
3444
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08003445 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003446 * If a page is a file cache, radix-tree replacement is very atomic
3447 * and we can skip this check. When it was an Anon page, its mapcount
3448 * goes down to 0. But because we added MIGRATION flage, it's not
3449 * uncharged yet. There are several case but page->mapcount check
3450 * and USED bit check in mem_cgroup_uncharge_page() will do enough
3451 * check. (see prepare_charge() also)
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08003452 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003453 if (PageAnon(used))
3454 mem_cgroup_uncharge_page(used);
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07003455 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003456 * At migration, we may charge account against cgroup which has no
3457 * tasks.
KAMEZAWA Hiroyuki88703262009-07-29 15:04:06 -07003458 * So, rmdir()->pre_destroy() can be called while we do this charge.
3459 * In that case, we need to call pre_destroy() again. check it here.
3460 */
3461 cgroup_release_and_wakeup_rmdir(&mem->css);
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08003462}
Pavel Emelianov78fb7462008-02-07 00:13:51 -08003463
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -07003464#ifdef CONFIG_DEBUG_VM
3465static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
3466{
3467 struct page_cgroup *pc;
3468
3469 pc = lookup_page_cgroup(page);
3470 if (likely(pc) && PageCgroupUsed(pc))
3471 return pc;
3472 return NULL;
3473}
3474
3475bool mem_cgroup_bad_page_check(struct page *page)
3476{
3477 if (mem_cgroup_disabled())
3478 return false;
3479
3480 return lookup_page_cgroup_used(page) != NULL;
3481}
3482
3483void mem_cgroup_print_bad_page(struct page *page)
3484{
3485 struct page_cgroup *pc;
3486
3487 pc = lookup_page_cgroup_used(page);
3488 if (pc) {
3489 int ret = -1;
3490 char *path;
3491
3492 printk(KERN_ALERT "pc:%p pc->flags:%lx pc->mem_cgroup:%p",
3493 pc, pc->flags, pc->mem_cgroup);
3494
3495 path = kmalloc(PATH_MAX, GFP_KERNEL);
3496 if (path) {
3497 rcu_read_lock();
3498 ret = cgroup_path(pc->mem_cgroup->css.cgroup,
3499 path, PATH_MAX);
3500 rcu_read_unlock();
3501 }
3502
3503 printk(KERN_CONT "(%s)\n",
3504 (ret < 0) ? "cannot get the path" : path);
3505 kfree(path);
3506 }
3507}
3508#endif
3509
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003510static DEFINE_MUTEX(set_limit_mutex);
3511
KOSAKI Motohirod38d2a72009-01-06 14:39:44 -08003512static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003513 unsigned long long val)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003514{
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003515 int retry_count;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003516 u64 memswlimit, memlimit;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003517 int ret = 0;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003518 int children = mem_cgroup_count_children(memcg);
3519 u64 curusage, oldusage;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003520 int enlarge;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003521
3522 /*
3523 * For keeping hierarchical_reclaim simple, how long we should retry
3524 * is depends on callers. We set our retry-count to be function
3525 * of # of children which we should visit in this loop.
3526 */
3527 retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
3528
3529 oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003530
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003531 enlarge = 0;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003532 while (retry_count) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003533 if (signal_pending(current)) {
3534 ret = -EINTR;
3535 break;
3536 }
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003537 /*
3538 * Rather than hide all in some function, I do this in
3539 * open coded manner. You see what this really does.
3540 * We have to guarantee mem->res.limit < mem->memsw.limit.
3541 */
3542 mutex_lock(&set_limit_mutex);
3543 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3544 if (memswlimit < val) {
3545 ret = -EINVAL;
3546 mutex_unlock(&set_limit_mutex);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003547 break;
3548 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003549
3550 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3551 if (memlimit < val)
3552 enlarge = 1;
3553
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003554 ret = res_counter_set_limit(&memcg->res, val);
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07003555 if (!ret) {
3556 if (memswlimit == val)
3557 memcg->memsw_is_minimum = true;
3558 else
3559 memcg->memsw_is_minimum = false;
3560 }
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003561 mutex_unlock(&set_limit_mutex);
3562
3563 if (!ret)
3564 break;
3565
Bob Liuaa20d482009-12-15 16:47:14 -08003566 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
Ying Han0ae5e892011-05-26 16:25:25 -07003567 MEM_CGROUP_RECLAIM_SHRINK,
3568 NULL);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003569 curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
3570 /* Usage is reduced ? */
3571 if (curusage >= oldusage)
3572 retry_count--;
3573 else
3574 oldusage = curusage;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003575 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003576 if (!ret && enlarge)
3577 memcg_oom_recover(memcg);
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08003578
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003579 return ret;
3580}
3581
Li Zefan338c8432009-06-17 16:27:15 -07003582static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
3583 unsigned long long val)
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003584{
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003585 int retry_count;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003586 u64 memlimit, memswlimit, oldusage, curusage;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003587 int children = mem_cgroup_count_children(memcg);
3588 int ret = -EBUSY;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003589 int enlarge = 0;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003590
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003591 /* see mem_cgroup_resize_res_limit */
3592 retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
3593 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003594 while (retry_count) {
3595 if (signal_pending(current)) {
3596 ret = -EINTR;
3597 break;
3598 }
3599 /*
3600 * Rather than hide all in some function, I do this in
3601 * open coded manner. You see what this really does.
3602 * We have to guarantee mem->res.limit < mem->memsw.limit.
3603 */
3604 mutex_lock(&set_limit_mutex);
3605 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3606 if (memlimit > val) {
3607 ret = -EINVAL;
3608 mutex_unlock(&set_limit_mutex);
3609 break;
3610 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003611 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3612 if (memswlimit < val)
3613 enlarge = 1;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003614 ret = res_counter_set_limit(&memcg->memsw, val);
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07003615 if (!ret) {
3616 if (memlimit == val)
3617 memcg->memsw_is_minimum = true;
3618 else
3619 memcg->memsw_is_minimum = false;
3620 }
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003621 mutex_unlock(&set_limit_mutex);
3622
3623 if (!ret)
3624 break;
3625
Balbir Singh4e416952009-09-23 15:56:39 -07003626 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
Balbir Singh75822b42009-09-23 15:56:38 -07003627 MEM_CGROUP_RECLAIM_NOSWAP |
Ying Han0ae5e892011-05-26 16:25:25 -07003628 MEM_CGROUP_RECLAIM_SHRINK,
3629 NULL);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003630 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003631 /* Usage is reduced ? */
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003632 if (curusage >= oldusage)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003633 retry_count--;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07003634 else
3635 oldusage = curusage;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003636 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003637 if (!ret && enlarge)
3638 memcg_oom_recover(memcg);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003639 return ret;
3640}
3641
Balbir Singh4e416952009-09-23 15:56:39 -07003642unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
Ying Han0ae5e892011-05-26 16:25:25 -07003643 gfp_t gfp_mask,
3644 unsigned long *total_scanned)
Balbir Singh4e416952009-09-23 15:56:39 -07003645{
3646 unsigned long nr_reclaimed = 0;
3647 struct mem_cgroup_per_zone *mz, *next_mz = NULL;
3648 unsigned long reclaimed;
3649 int loop = 0;
3650 struct mem_cgroup_tree_per_zone *mctz;
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07003651 unsigned long long excess;
Ying Han0ae5e892011-05-26 16:25:25 -07003652 unsigned long nr_scanned;
Balbir Singh4e416952009-09-23 15:56:39 -07003653
3654 if (order > 0)
3655 return 0;
3656
KOSAKI Motohiro00918b62010-08-10 18:03:05 -07003657 mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
Balbir Singh4e416952009-09-23 15:56:39 -07003658 /*
3659 * This loop can run a while, specially if mem_cgroup's continuously
3660 * keep exceeding their soft limit and putting the system under
3661 * pressure
3662 */
3663 do {
3664 if (next_mz)
3665 mz = next_mz;
3666 else
3667 mz = mem_cgroup_largest_soft_limit_node(mctz);
3668 if (!mz)
3669 break;
3670
Ying Han0ae5e892011-05-26 16:25:25 -07003671 nr_scanned = 0;
Balbir Singh4e416952009-09-23 15:56:39 -07003672 reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
3673 gfp_mask,
Ying Han0ae5e892011-05-26 16:25:25 -07003674 MEM_CGROUP_RECLAIM_SOFT,
3675 &nr_scanned);
Balbir Singh4e416952009-09-23 15:56:39 -07003676 nr_reclaimed += reclaimed;
Ying Han0ae5e892011-05-26 16:25:25 -07003677 *total_scanned += nr_scanned;
Balbir Singh4e416952009-09-23 15:56:39 -07003678 spin_lock(&mctz->lock);
3679
3680 /*
3681 * If we failed to reclaim anything from this memory cgroup
3682 * it is time to move on to the next cgroup
3683 */
3684 next_mz = NULL;
3685 if (!reclaimed) {
3686 do {
3687 /*
3688 * Loop until we find yet another one.
3689 *
3690 * By the time we get the soft_limit lock
3691 * again, someone might have aded the
3692 * group back on the RB tree. Iterate to
3693 * make sure we get a different mem.
3694 * mem_cgroup_largest_soft_limit_node returns
3695 * NULL if no other cgroup is present on
3696 * the tree
3697 */
3698 next_mz =
3699 __mem_cgroup_largest_soft_limit_node(mctz);
Michal Hocko39cc98f2011-05-26 16:25:28 -07003700 if (next_mz == mz)
Balbir Singh4e416952009-09-23 15:56:39 -07003701 css_put(&next_mz->mem->css);
Michal Hocko39cc98f2011-05-26 16:25:28 -07003702 else /* next_mz == NULL or other memcg */
Balbir Singh4e416952009-09-23 15:56:39 -07003703 break;
3704 } while (1);
3705 }
Balbir Singh4e416952009-09-23 15:56:39 -07003706 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07003707 excess = res_counter_soft_limit_excess(&mz->mem->res);
Balbir Singh4e416952009-09-23 15:56:39 -07003708 /*
3709 * One school of thought says that we should not add
3710 * back the node to the tree if reclaim returns 0.
3711 * But our reclaim could return 0, simply because due
3712 * to priority we are exposing a smaller subset of
3713 * memory to reclaim from. Consider this as a longer
3714 * term TODO.
3715 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07003716 /* If excess == 0, no tree ops */
3717 __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
Balbir Singh4e416952009-09-23 15:56:39 -07003718 spin_unlock(&mctz->lock);
3719 css_put(&mz->mem->css);
3720 loop++;
3721 /*
3722 * Could not reclaim anything and there are no more
3723 * mem cgroups to try or we seem to be looping without
3724 * reclaiming anything.
3725 */
3726 if (!nr_reclaimed &&
3727 (next_mz == NULL ||
3728 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
3729 break;
3730 } while (!nr_reclaimed);
3731 if (next_mz)
3732 css_put(&next_mz->mem->css);
3733 return nr_reclaimed;
3734}
3735
KAMEZAWA Hiroyukic9b0ed52008-07-25 01:47:15 -07003736/*
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003737 * This routine traverse page_cgroup in given list and drop them all.
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003738 * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
3739 */
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003740static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003741 int node, int zid, enum lru_list lru)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003742{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003743 struct zone *zone;
3744 struct mem_cgroup_per_zone *mz;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003745 struct page_cgroup *pc, *busy;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003746 unsigned long flags, loop;
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -08003747 struct list_head *list;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003748 int ret = 0;
KAMEZAWA Hiroyuki072c56c2008-02-07 00:14:39 -08003749
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003750 zone = &NODE_DATA(node)->node_zones[zid];
3751 mz = mem_cgroup_zoneinfo(mem, node, zid);
Christoph Lameterb69408e2008-10-18 20:26:14 -07003752 list = &mz->lists[lru];
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003753
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003754 loop = MEM_CGROUP_ZSTAT(mz, lru);
3755 /* give some margin against EBUSY etc...*/
3756 loop += 256;
3757 busy = NULL;
3758 while (loop--) {
Johannes Weiner5564e882011-03-23 16:42:29 -07003759 struct page *page;
3760
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003761 ret = 0;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003762 spin_lock_irqsave(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003763 if (list_empty(list)) {
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003764 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003765 break;
3766 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003767 pc = list_entry(list->prev, struct page_cgroup, lru);
3768 if (busy == pc) {
3769 list_move(&pc->lru, list);
Thiago Farina648bcc72010-03-05 13:42:04 -08003770 busy = NULL;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003771 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003772 continue;
3773 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003774 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003775
Johannes Weiner6b3ae582011-03-23 16:42:30 -07003776 page = lookup_cgroup_page(pc);
Johannes Weiner5564e882011-03-23 16:42:29 -07003777
3778 ret = mem_cgroup_move_parent(page, pc, mem, GFP_KERNEL);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003779 if (ret == -ENOMEM)
3780 break;
3781
3782 if (ret == -EBUSY || ret == -EINVAL) {
3783 /* found lock contention or "pc" is obsolete. */
3784 busy = pc;
3785 cond_resched();
3786 } else
3787 busy = NULL;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003788 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003789
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003790 if (!ret && !list_empty(list))
3791 return -EBUSY;
3792 return ret;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003793}
3794
3795/*
3796 * make mem_cgroup's charge to be 0 if there is no task.
3797 * This enables deleting this mem_cgroup.
3798 */
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003799static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003800{
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003801 int ret;
3802 int node, zid, shrink;
3803 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003804 struct cgroup *cgrp = mem->css.cgroup;
Hugh Dickins8869b8f2008-03-04 14:29:09 -08003805
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003806 css_get(&mem->css);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003807
3808 shrink = 0;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003809 /* should free all ? */
3810 if (free_all)
3811 goto try_to_free;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003812move_account:
Daisuke Nishimurafce66472010-01-15 17:01:30 -08003813 do {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003814 ret = -EBUSY;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003815 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003816 goto out;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003817 ret = -EINTR;
3818 if (signal_pending(current))
3819 goto out;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003820 /* This is for making all *used* pages to be on LRU. */
3821 lru_add_drain_all();
Michal Hockod38144b2011-07-26 16:08:28 -07003822 drain_all_stock_sync(mem);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003823 ret = 0;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07003824 mem_cgroup_start_move(mem);
KAMEZAWA Hiroyuki299b4ea2009-01-29 14:25:17 -08003825 for_each_node_state(node, N_HIGH_MEMORY) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003826 for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
Christoph Lameterb69408e2008-10-18 20:26:14 -07003827 enum lru_list l;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003828 for_each_lru(l) {
3829 ret = mem_cgroup_force_empty_list(mem,
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003830 node, zid, l);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003831 if (ret)
3832 break;
3833 }
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08003834 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003835 if (ret)
3836 break;
3837 }
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07003838 mem_cgroup_end_move(mem);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003839 memcg_oom_recover(mem);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003840 /* it seems parent cgroup doesn't have enough mem */
3841 if (ret == -ENOMEM)
3842 goto try_to_free;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003843 cond_resched();
Daisuke Nishimurafce66472010-01-15 17:01:30 -08003844 /* "ret" should also be checked to ensure all lists are empty. */
3845 } while (mem->res.usage > 0 || ret);
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003846out:
3847 css_put(&mem->css);
3848 return ret;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003849
3850try_to_free:
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003851 /* returns EBUSY if there is a task or if we come here twice. */
3852 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003853 ret = -EBUSY;
3854 goto out;
3855 }
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003856 /* we call try-to-free pages for make this cgroup empty */
3857 lru_add_drain_all();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003858 /* try to free all pages in this cgroup */
3859 shrink = 1;
3860 while (nr_retries && mem->res.usage > 0) {
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07003861 struct memcg_scanrecord rec;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003862 int progress;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003863
3864 if (signal_pending(current)) {
3865 ret = -EINTR;
3866 goto out;
3867 }
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07003868 rec.context = SCAN_BY_SHRINK;
3869 rec.mem = mem;
3870 rec.root = mem;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08003871 progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07003872 false, &rec);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003873 if (!progress) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003874 nr_retries--;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003875 /* maybe some writeback is necessary */
Jens Axboe8aa7e842009-07-09 14:52:32 +02003876 congestion_wait(BLK_RW_ASYNC, HZ/10);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003877 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003878
3879 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003880 lru_add_drain();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003881 /* try move_account...there may be some *locked* pages. */
Daisuke Nishimurafce66472010-01-15 17:01:30 -08003882 goto move_account;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08003883}
3884
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08003885int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
3886{
3887 return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
3888}
3889
3890
Balbir Singh18f59ea2009-01-07 18:08:07 -08003891static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
3892{
3893 return mem_cgroup_from_cont(cont)->use_hierarchy;
3894}
3895
3896static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
3897 u64 val)
3898{
3899 int retval = 0;
3900 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
3901 struct cgroup *parent = cont->parent;
3902 struct mem_cgroup *parent_mem = NULL;
3903
3904 if (parent)
3905 parent_mem = mem_cgroup_from_cont(parent);
3906
3907 cgroup_lock();
3908 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02003909 * If parent's use_hierarchy is set, we can't make any modifications
Balbir Singh18f59ea2009-01-07 18:08:07 -08003910 * in the child subtrees. If it is unset, then the change can
3911 * occur, provided the current cgroup has no children.
3912 *
3913 * For the root cgroup, parent_mem is NULL, we allow value to be
3914 * set if there are no children.
3915 */
3916 if ((!parent_mem || !parent_mem->use_hierarchy) &&
3917 (val == 1 || val == 0)) {
3918 if (list_empty(&cont->children))
3919 mem->use_hierarchy = val;
3920 else
3921 retval = -EBUSY;
3922 } else
3923 retval = -EINVAL;
3924 cgroup_unlock();
3925
3926 return retval;
3927}
3928
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003929
Johannes Weiner7a159cc2011-03-23 16:42:38 -07003930static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *mem,
3931 enum mem_cgroup_stat_index idx)
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003932{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003933 struct mem_cgroup *iter;
Johannes Weiner7a159cc2011-03-23 16:42:38 -07003934 long val = 0;
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003935
Johannes Weiner7a159cc2011-03-23 16:42:38 -07003936 /* Per-cpu values can be negative, use a signed accumulator */
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003937 for_each_mem_cgroup_tree(iter, mem)
3938 val += mem_cgroup_read_stat(iter, idx);
3939
3940 if (val < 0) /* race ? */
3941 val = 0;
3942 return val;
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003943}
3944
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003945static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
3946{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003947 u64 val;
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003948
3949 if (!mem_cgroup_is_root(mem)) {
3950 if (!swap)
3951 return res_counter_read_u64(&mem->res, RES_USAGE);
3952 else
3953 return res_counter_read_u64(&mem->memsw, RES_USAGE);
3954 }
3955
Johannes Weiner7a159cc2011-03-23 16:42:38 -07003956 val = mem_cgroup_recursive_stat(mem, MEM_CGROUP_STAT_CACHE);
3957 val += mem_cgroup_recursive_stat(mem, MEM_CGROUP_STAT_RSS);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003958
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07003959 if (swap)
Johannes Weiner7a159cc2011-03-23 16:42:38 -07003960 val += mem_cgroup_recursive_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003961
3962 return val << PAGE_SHIFT;
3963}
3964
Paul Menage2c3daa72008-04-29 00:59:58 -07003965static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003966{
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003967 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003968 u64 val;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003969 int type, name;
3970
3971 type = MEMFILE_TYPE(cft->private);
3972 name = MEMFILE_ATTR(cft->private);
3973 switch (type) {
3974 case _MEM:
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003975 if (name == RES_USAGE)
3976 val = mem_cgroup_usage(mem, false);
3977 else
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003978 val = res_counter_read_u64(&mem->res, name);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003979 break;
3980 case _MEMSWAP:
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08003981 if (name == RES_USAGE)
3982 val = mem_cgroup_usage(mem, true);
3983 else
Balbir Singh0c3e73e2009-09-23 15:56:42 -07003984 val = res_counter_read_u64(&mem->memsw, name);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08003985 break;
3986 default:
3987 BUG();
3988 break;
3989 }
3990 return val;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003991}
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003992/*
3993 * The user of this function is...
3994 * RES_LIMIT.
3995 */
Paul Menage856c13a2008-07-25 01:47:04 -07003996static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
3997 const char *buffer)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08003998{
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07003999 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004000 int type, name;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004001 unsigned long long val;
4002 int ret;
4003
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004004 type = MEMFILE_TYPE(cft->private);
4005 name = MEMFILE_ATTR(cft->private);
4006 switch (name) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004007 case RES_LIMIT:
Balbir Singh4b3bde42009-09-23 15:56:32 -07004008 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
4009 ret = -EINVAL;
4010 break;
4011 }
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004012 /* This function does all necessary parse...reuse it */
4013 ret = res_counter_memparse_write_strategy(buffer, &val);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004014 if (ret)
4015 break;
4016 if (type == _MEM)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004017 ret = mem_cgroup_resize_limit(memcg, val);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004018 else
4019 ret = mem_cgroup_resize_memsw_limit(memcg, val);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004020 break;
Balbir Singh296c81d2009-09-23 15:56:36 -07004021 case RES_SOFT_LIMIT:
4022 ret = res_counter_memparse_write_strategy(buffer, &val);
4023 if (ret)
4024 break;
4025 /*
4026 * For memsw, soft limits are hard to implement in terms
4027 * of semantics, for now, we support soft limits for
4028 * control without swap
4029 */
4030 if (type == _MEM)
4031 ret = res_counter_set_soft_limit(&memcg->res, val);
4032 else
4033 ret = -EINVAL;
4034 break;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004035 default:
4036 ret = -EINVAL; /* should be BUG() ? */
4037 break;
4038 }
4039 return ret;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004040}
4041
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08004042static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
4043 unsigned long long *mem_limit, unsigned long long *memsw_limit)
4044{
4045 struct cgroup *cgroup;
4046 unsigned long long min_limit, min_memsw_limit, tmp;
4047
4048 min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4049 min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4050 cgroup = memcg->css.cgroup;
4051 if (!memcg->use_hierarchy)
4052 goto out;
4053
4054 while (cgroup->parent) {
4055 cgroup = cgroup->parent;
4056 memcg = mem_cgroup_from_cont(cgroup);
4057 if (!memcg->use_hierarchy)
4058 break;
4059 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
4060 min_limit = min(min_limit, tmp);
4061 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4062 min_memsw_limit = min(min_memsw_limit, tmp);
4063 }
4064out:
4065 *mem_limit = min_limit;
4066 *memsw_limit = min_memsw_limit;
4067 return;
4068}
4069
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07004070static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004071{
4072 struct mem_cgroup *mem;
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004073 int type, name;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004074
4075 mem = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004076 type = MEMFILE_TYPE(event);
4077 name = MEMFILE_ATTR(event);
4078 switch (name) {
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07004079 case RES_MAX_USAGE:
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004080 if (type == _MEM)
4081 res_counter_reset_max(&mem->res);
4082 else
4083 res_counter_reset_max(&mem->memsw);
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07004084 break;
4085 case RES_FAILCNT:
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004086 if (type == _MEM)
4087 res_counter_reset_failcnt(&mem->res);
4088 else
4089 res_counter_reset_failcnt(&mem->memsw);
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07004090 break;
4091 }
Balbir Singhf64c3f52009-09-23 15:56:37 -07004092
Pavel Emelyanov85cc59d2008-04-29 01:00:20 -07004093 return 0;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004094}
4095
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004096static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
4097 struct cftype *cft)
4098{
4099 return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
4100}
4101
Daisuke Nishimura02491442010-03-10 15:22:17 -08004102#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004103static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
4104 struct cftype *cft, u64 val)
4105{
4106 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4107
4108 if (val >= (1 << NR_MOVE_TYPE))
4109 return -EINVAL;
4110 /*
4111 * We check this value several times in both in can_attach() and
4112 * attach(), so we need cgroup lock to prevent this value from being
4113 * inconsistent.
4114 */
4115 cgroup_lock();
4116 mem->move_charge_at_immigrate = val;
4117 cgroup_unlock();
4118
4119 return 0;
4120}
Daisuke Nishimura02491442010-03-10 15:22:17 -08004121#else
4122static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
4123 struct cftype *cft, u64 val)
4124{
4125 return -ENOSYS;
4126}
4127#endif
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004128
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004129
4130/* For read statistics */
4131enum {
4132 MCS_CACHE,
4133 MCS_RSS,
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -08004134 MCS_FILE_MAPPED,
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004135 MCS_PGPGIN,
4136 MCS_PGPGOUT,
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07004137 MCS_SWAP,
Ying Han456f9982011-05-26 16:25:38 -07004138 MCS_PGFAULT,
4139 MCS_PGMAJFAULT,
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004140 MCS_INACTIVE_ANON,
4141 MCS_ACTIVE_ANON,
4142 MCS_INACTIVE_FILE,
4143 MCS_ACTIVE_FILE,
4144 MCS_UNEVICTABLE,
4145 NR_MCS_STAT,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004146};
4147
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004148struct mcs_total_stat {
4149 s64 stat[NR_MCS_STAT];
4150};
4151
4152struct {
4153 char *local_name;
4154 char *total_name;
4155} memcg_stat_strings[NR_MCS_STAT] = {
4156 {"cache", "total_cache"},
4157 {"rss", "total_rss"},
Balbir Singhd69b0422009-06-17 16:26:34 -07004158 {"mapped_file", "total_mapped_file"},
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004159 {"pgpgin", "total_pgpgin"},
4160 {"pgpgout", "total_pgpgout"},
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07004161 {"swap", "total_swap"},
Ying Han456f9982011-05-26 16:25:38 -07004162 {"pgfault", "total_pgfault"},
4163 {"pgmajfault", "total_pgmajfault"},
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004164 {"inactive_anon", "total_inactive_anon"},
4165 {"active_anon", "total_active_anon"},
4166 {"inactive_file", "total_inactive_file"},
4167 {"active_file", "total_active_file"},
4168 {"unevictable", "total_unevictable"}
4169};
4170
4171
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07004172static void
4173mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004174{
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004175 s64 val;
4176
4177 /* per cpu stat */
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004178 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004179 s->stat[MCS_CACHE] += val * PAGE_SIZE;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004180 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004181 s->stat[MCS_RSS] += val * PAGE_SIZE;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004182 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED);
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -08004183 s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
Johannes Weinere9f89742011-03-23 16:42:37 -07004184 val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGPGIN);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004185 s->stat[MCS_PGPGIN] += val;
Johannes Weinere9f89742011-03-23 16:42:37 -07004186 val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGPGOUT);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004187 s->stat[MCS_PGPGOUT] += val;
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07004188 if (do_swap_account) {
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004189 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07004190 s->stat[MCS_SWAP] += val * PAGE_SIZE;
4191 }
Ying Han456f9982011-05-26 16:25:38 -07004192 val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGFAULT);
4193 s->stat[MCS_PGFAULT] += val;
4194 val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGMAJFAULT);
4195 s->stat[MCS_PGMAJFAULT] += val;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004196
4197 /* per zone stat */
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004198 val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_INACTIVE_ANON));
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004199 s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004200 val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_ACTIVE_ANON));
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004201 s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004202 val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_INACTIVE_FILE));
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004203 s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004204 val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_ACTIVE_FILE));
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004205 s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004206 val = mem_cgroup_nr_lru_pages(mem, BIT(LRU_UNEVICTABLE));
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004207 s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004208}
4209
4210static void
4211mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
4212{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07004213 struct mem_cgroup *iter;
4214
4215 for_each_mem_cgroup_tree(iter, mem)
4216 mem_cgroup_get_local_stat(iter, s);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004217}
4218
Ying Han406eb0c2011-05-26 16:25:37 -07004219#ifdef CONFIG_NUMA
4220static int mem_control_numa_stat_show(struct seq_file *m, void *arg)
4221{
4222 int nid;
4223 unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
4224 unsigned long node_nr;
4225 struct cgroup *cont = m->private;
4226 struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
4227
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004228 total_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL);
Ying Han406eb0c2011-05-26 16:25:37 -07004229 seq_printf(m, "total=%lu", total_nr);
4230 for_each_node_state(nid, N_HIGH_MEMORY) {
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004231 node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid, LRU_ALL);
Ying Han406eb0c2011-05-26 16:25:37 -07004232 seq_printf(m, " N%d=%lu", nid, node_nr);
4233 }
4234 seq_putc(m, '\n');
4235
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004236 file_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_FILE);
Ying Han406eb0c2011-05-26 16:25:37 -07004237 seq_printf(m, "file=%lu", file_nr);
4238 for_each_node_state(nid, N_HIGH_MEMORY) {
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004239 node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
4240 LRU_ALL_FILE);
Ying Han406eb0c2011-05-26 16:25:37 -07004241 seq_printf(m, " N%d=%lu", nid, node_nr);
4242 }
4243 seq_putc(m, '\n');
4244
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004245 anon_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_ANON);
Ying Han406eb0c2011-05-26 16:25:37 -07004246 seq_printf(m, "anon=%lu", anon_nr);
4247 for_each_node_state(nid, N_HIGH_MEMORY) {
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004248 node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
4249 LRU_ALL_ANON);
Ying Han406eb0c2011-05-26 16:25:37 -07004250 seq_printf(m, " N%d=%lu", nid, node_nr);
4251 }
4252 seq_putc(m, '\n');
4253
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004254 unevictable_nr = mem_cgroup_nr_lru_pages(mem_cont, BIT(LRU_UNEVICTABLE));
Ying Han406eb0c2011-05-26 16:25:37 -07004255 seq_printf(m, "unevictable=%lu", unevictable_nr);
4256 for_each_node_state(nid, N_HIGH_MEMORY) {
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07004257 node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
4258 BIT(LRU_UNEVICTABLE));
Ying Han406eb0c2011-05-26 16:25:37 -07004259 seq_printf(m, " N%d=%lu", nid, node_nr);
4260 }
4261 seq_putc(m, '\n');
4262 return 0;
4263}
4264#endif /* CONFIG_NUMA */
4265
Paul Menagec64745c2008-04-29 01:00:02 -07004266static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
4267 struct cgroup_map_cb *cb)
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004268{
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004269 struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004270 struct mcs_total_stat mystat;
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004271 int i;
4272
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004273 memset(&mystat, 0, sizeof(mystat));
4274 mem_cgroup_get_local_stat(mem_cont, &mystat);
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004275
Ying Han406eb0c2011-05-26 16:25:37 -07004276
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07004277 for (i = 0; i < NR_MCS_STAT; i++) {
4278 if (i == MCS_SWAP && !do_swap_account)
4279 continue;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004280 cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07004281 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004282
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004283 /* Hierarchical information */
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08004284 {
4285 unsigned long long limit, memsw_limit;
4286 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
4287 cb->fill(cb, "hierarchical_memory_limit", limit);
4288 if (do_swap_account)
4289 cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
4290 }
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004291
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004292 memset(&mystat, 0, sizeof(mystat));
4293 mem_cgroup_get_total_stat(mem_cont, &mystat);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07004294 for (i = 0; i < NR_MCS_STAT; i++) {
4295 if (i == MCS_SWAP && !do_swap_account)
4296 continue;
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004297 cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07004298 }
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07004299
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004300#ifdef CONFIG_DEBUG_VM
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08004301 cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08004302
4303 {
4304 int nid, zid;
4305 struct mem_cgroup_per_zone *mz;
4306 unsigned long recent_rotated[2] = {0, 0};
4307 unsigned long recent_scanned[2] = {0, 0};
4308
4309 for_each_online_node(nid)
4310 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
4311 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
4312
4313 recent_rotated[0] +=
4314 mz->reclaim_stat.recent_rotated[0];
4315 recent_rotated[1] +=
4316 mz->reclaim_stat.recent_rotated[1];
4317 recent_scanned[0] +=
4318 mz->reclaim_stat.recent_scanned[0];
4319 recent_scanned[1] +=
4320 mz->reclaim_stat.recent_scanned[1];
4321 }
4322 cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
4323 cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
4324 cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
4325 cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
4326 }
4327#endif
4328
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004329 return 0;
4330}
4331
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004332static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
4333{
4334 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4335
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07004336 return mem_cgroup_swappiness(memcg);
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004337}
4338
4339static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
4340 u64 val)
4341{
4342 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4343 struct mem_cgroup *parent;
Li Zefan068b38c2009-01-15 13:51:26 -08004344
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004345 if (val > 100)
4346 return -EINVAL;
4347
4348 if (cgrp->parent == NULL)
4349 return -EINVAL;
4350
4351 parent = mem_cgroup_from_cont(cgrp->parent);
Li Zefan068b38c2009-01-15 13:51:26 -08004352
4353 cgroup_lock();
4354
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004355 /* If under hierarchy, only empty-root can set this value */
4356 if ((parent->use_hierarchy) ||
Li Zefan068b38c2009-01-15 13:51:26 -08004357 (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
4358 cgroup_unlock();
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004359 return -EINVAL;
Li Zefan068b38c2009-01-15 13:51:26 -08004360 }
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004361
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004362 memcg->swappiness = val;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004363
Li Zefan068b38c2009-01-15 13:51:26 -08004364 cgroup_unlock();
4365
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004366 return 0;
4367}
4368
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004369static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
4370{
4371 struct mem_cgroup_threshold_ary *t;
4372 u64 usage;
4373 int i;
4374
4375 rcu_read_lock();
4376 if (!swap)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004377 t = rcu_dereference(memcg->thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004378 else
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004379 t = rcu_dereference(memcg->memsw_thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004380
4381 if (!t)
4382 goto unlock;
4383
4384 usage = mem_cgroup_usage(memcg, swap);
4385
4386 /*
4387 * current_threshold points to threshold just below usage.
4388 * If it's not true, a threshold was crossed after last
4389 * call of __mem_cgroup_threshold().
4390 */
Phil Carmody5407a562010-05-26 14:42:42 -07004391 i = t->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004392
4393 /*
4394 * Iterate backward over array of thresholds starting from
4395 * current_threshold and check if a threshold is crossed.
4396 * If none of thresholds below usage is crossed, we read
4397 * only one element of the array here.
4398 */
4399 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
4400 eventfd_signal(t->entries[i].eventfd, 1);
4401
4402 /* i = current_threshold + 1 */
4403 i++;
4404
4405 /*
4406 * Iterate forward over array of thresholds starting from
4407 * current_threshold+1 and check if a threshold is crossed.
4408 * If none of thresholds above usage is crossed, we read
4409 * only one element of the array here.
4410 */
4411 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
4412 eventfd_signal(t->entries[i].eventfd, 1);
4413
4414 /* Update current_threshold */
Phil Carmody5407a562010-05-26 14:42:42 -07004415 t->current_threshold = i - 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004416unlock:
4417 rcu_read_unlock();
4418}
4419
4420static void mem_cgroup_threshold(struct mem_cgroup *memcg)
4421{
Kirill A. Shutemovad4ca5f2010-10-07 12:59:27 -07004422 while (memcg) {
4423 __mem_cgroup_threshold(memcg, false);
4424 if (do_swap_account)
4425 __mem_cgroup_threshold(memcg, true);
4426
4427 memcg = parent_mem_cgroup(memcg);
4428 }
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004429}
4430
4431static int compare_thresholds(const void *a, const void *b)
4432{
4433 const struct mem_cgroup_threshold *_a = a;
4434 const struct mem_cgroup_threshold *_b = b;
4435
4436 return _a->threshold - _b->threshold;
4437}
4438
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07004439static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004440{
4441 struct mem_cgroup_eventfd_list *ev;
4442
4443 list_for_each_entry(ev, &mem->oom_notify, list)
4444 eventfd_signal(ev->eventfd, 1);
4445 return 0;
4446}
4447
4448static void mem_cgroup_oom_notify(struct mem_cgroup *mem)
4449{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07004450 struct mem_cgroup *iter;
4451
4452 for_each_mem_cgroup_tree(iter, mem)
4453 mem_cgroup_oom_notify_cb(iter);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004454}
4455
4456static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
4457 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004458{
4459 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004460 struct mem_cgroup_thresholds *thresholds;
4461 struct mem_cgroup_threshold_ary *new;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004462 int type = MEMFILE_TYPE(cft->private);
4463 u64 threshold, usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004464 int i, size, ret;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004465
4466 ret = res_counter_memparse_write_strategy(args, &threshold);
4467 if (ret)
4468 return ret;
4469
4470 mutex_lock(&memcg->thresholds_lock);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004471
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004472 if (type == _MEM)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004473 thresholds = &memcg->thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004474 else if (type == _MEMSWAP)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004475 thresholds = &memcg->memsw_thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004476 else
4477 BUG();
4478
4479 usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
4480
4481 /* Check if a threshold crossed before adding a new one */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004482 if (thresholds->primary)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004483 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4484
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004485 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004486
4487 /* Allocate memory for new array of thresholds */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004488 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004489 GFP_KERNEL);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004490 if (!new) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004491 ret = -ENOMEM;
4492 goto unlock;
4493 }
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004494 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004495
4496 /* Copy thresholds (if any) to new array */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004497 if (thresholds->primary) {
4498 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004499 sizeof(struct mem_cgroup_threshold));
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004500 }
4501
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004502 /* Add new threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004503 new->entries[size - 1].eventfd = eventfd;
4504 new->entries[size - 1].threshold = threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004505
4506 /* Sort thresholds. Registering of new threshold isn't time-critical */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004507 sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004508 compare_thresholds, NULL);
4509
4510 /* Find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004511 new->current_threshold = -1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004512 for (i = 0; i < size; i++) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004513 if (new->entries[i].threshold < usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004514 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004515 * new->current_threshold will not be used until
4516 * rcu_assign_pointer(), so it's safe to increment
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004517 * it here.
4518 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004519 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004520 }
4521 }
4522
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004523 /* Free old spare buffer and save old primary buffer as spare */
4524 kfree(thresholds->spare);
4525 thresholds->spare = thresholds->primary;
4526
4527 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004528
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004529 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004530 synchronize_rcu();
4531
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004532unlock:
4533 mutex_unlock(&memcg->thresholds_lock);
4534
4535 return ret;
4536}
4537
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004538static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004539 struct cftype *cft, struct eventfd_ctx *eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004540{
4541 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004542 struct mem_cgroup_thresholds *thresholds;
4543 struct mem_cgroup_threshold_ary *new;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004544 int type = MEMFILE_TYPE(cft->private);
4545 u64 usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004546 int i, j, size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004547
4548 mutex_lock(&memcg->thresholds_lock);
4549 if (type == _MEM)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004550 thresholds = &memcg->thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004551 else if (type == _MEMSWAP)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004552 thresholds = &memcg->memsw_thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004553 else
4554 BUG();
4555
4556 /*
4557 * Something went wrong if we trying to unregister a threshold
4558 * if we don't have thresholds
4559 */
4560 BUG_ON(!thresholds);
4561
4562 usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
4563
4564 /* Check if a threshold crossed before removing */
4565 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
4566
4567 /* Calculate new number of threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004568 size = 0;
4569 for (i = 0; i < thresholds->primary->size; i++) {
4570 if (thresholds->primary->entries[i].eventfd != eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004571 size++;
4572 }
4573
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004574 new = thresholds->spare;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004575
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004576 /* Set thresholds array to NULL if we don't have thresholds */
4577 if (!size) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004578 kfree(new);
4579 new = NULL;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004580 goto swap_buffers;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004581 }
4582
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004583 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004584
4585 /* Copy thresholds and find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004586 new->current_threshold = -1;
4587 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
4588 if (thresholds->primary->entries[i].eventfd == eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004589 continue;
4590
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004591 new->entries[j] = thresholds->primary->entries[i];
4592 if (new->entries[j].threshold < usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004593 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004594 * new->current_threshold will not be used
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004595 * until rcu_assign_pointer(), so it's safe to increment
4596 * it here.
4597 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004598 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004599 }
4600 j++;
4601 }
4602
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004603swap_buffers:
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07004604 /* Swap primary and spare array */
4605 thresholds->spare = thresholds->primary;
4606 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004607
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004608 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004609 synchronize_rcu();
4610
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004611 mutex_unlock(&memcg->thresholds_lock);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08004612}
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004613
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004614static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
4615 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
4616{
4617 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
4618 struct mem_cgroup_eventfd_list *event;
4619 int type = MEMFILE_TYPE(cft->private);
4620
4621 BUG_ON(type != _OOM_TYPE);
4622 event = kmalloc(sizeof(*event), GFP_KERNEL);
4623 if (!event)
4624 return -ENOMEM;
4625
Michal Hocko1af8efe2011-07-26 16:08:24 -07004626 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004627
4628 event->eventfd = eventfd;
4629 list_add(&event->list, &memcg->oom_notify);
4630
4631 /* already in OOM ? */
Michal Hocko79dfdac2011-07-26 16:08:23 -07004632 if (atomic_read(&memcg->under_oom))
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004633 eventfd_signal(eventfd, 1);
Michal Hocko1af8efe2011-07-26 16:08:24 -07004634 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004635
4636 return 0;
4637}
4638
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07004639static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004640 struct cftype *cft, struct eventfd_ctx *eventfd)
4641{
4642 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4643 struct mem_cgroup_eventfd_list *ev, *tmp;
4644 int type = MEMFILE_TYPE(cft->private);
4645
4646 BUG_ON(type != _OOM_TYPE);
4647
Michal Hocko1af8efe2011-07-26 16:08:24 -07004648 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004649
4650 list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) {
4651 if (ev->eventfd == eventfd) {
4652 list_del(&ev->list);
4653 kfree(ev);
4654 }
4655 }
4656
Michal Hocko1af8efe2011-07-26 16:08:24 -07004657 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004658}
4659
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004660static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
4661 struct cftype *cft, struct cgroup_map_cb *cb)
4662{
4663 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4664
4665 cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable);
4666
Michal Hocko79dfdac2011-07-26 16:08:23 -07004667 if (atomic_read(&mem->under_oom))
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004668 cb->fill(cb, "under_oom", 1);
4669 else
4670 cb->fill(cb, "under_oom", 0);
4671 return 0;
4672}
4673
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004674static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
4675 struct cftype *cft, u64 val)
4676{
4677 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4678 struct mem_cgroup *parent;
4679
4680 /* cannot set to root cgroup and only 0 and 1 are allowed */
4681 if (!cgrp->parent || !((val == 0) || (val == 1)))
4682 return -EINVAL;
4683
4684 parent = mem_cgroup_from_cont(cgrp->parent);
4685
4686 cgroup_lock();
4687 /* oom-kill-disable is a flag for subhierarchy. */
4688 if ((parent->use_hierarchy) ||
4689 (mem->use_hierarchy && !list_empty(&cgrp->children))) {
4690 cgroup_unlock();
4691 return -EINVAL;
4692 }
4693 mem->oom_kill_disable = val;
KAMEZAWA Hiroyuki4d845eb2010-06-29 15:05:18 -07004694 if (!val)
4695 memcg_oom_recover(mem);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004696 cgroup_unlock();
4697 return 0;
4698}
4699
Ying Han406eb0c2011-05-26 16:25:37 -07004700#ifdef CONFIG_NUMA
4701static const struct file_operations mem_control_numa_stat_file_operations = {
4702 .read = seq_read,
4703 .llseek = seq_lseek,
4704 .release = single_release,
4705};
4706
4707static int mem_control_numa_stat_open(struct inode *unused, struct file *file)
4708{
4709 struct cgroup *cont = file->f_dentry->d_parent->d_fsdata;
4710
4711 file->f_op = &mem_control_numa_stat_file_operations;
4712 return single_open(file, mem_control_numa_stat_show, cont);
4713}
4714#endif /* CONFIG_NUMA */
4715
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07004716static int mem_cgroup_vmscan_stat_read(struct cgroup *cgrp,
4717 struct cftype *cft,
4718 struct cgroup_map_cb *cb)
4719{
4720 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4721 char string[64];
4722 int i;
4723
4724 for (i = 0; i < NR_SCANSTATS; i++) {
4725 strcpy(string, scanstat_string[i]);
4726 strcat(string, SCANSTAT_WORD_LIMIT);
4727 cb->fill(cb, string, mem->scanstat.stats[SCAN_BY_LIMIT][i]);
4728 }
4729
4730 for (i = 0; i < NR_SCANSTATS; i++) {
4731 strcpy(string, scanstat_string[i]);
4732 strcat(string, SCANSTAT_WORD_SYSTEM);
4733 cb->fill(cb, string, mem->scanstat.stats[SCAN_BY_SYSTEM][i]);
4734 }
4735
4736 for (i = 0; i < NR_SCANSTATS; i++) {
4737 strcpy(string, scanstat_string[i]);
4738 strcat(string, SCANSTAT_WORD_LIMIT);
4739 strcat(string, SCANSTAT_WORD_HIERARCHY);
4740 cb->fill(cb, string, mem->scanstat.rootstats[SCAN_BY_LIMIT][i]);
4741 }
4742 for (i = 0; i < NR_SCANSTATS; i++) {
4743 strcpy(string, scanstat_string[i]);
4744 strcat(string, SCANSTAT_WORD_SYSTEM);
4745 strcat(string, SCANSTAT_WORD_HIERARCHY);
4746 cb->fill(cb, string, mem->scanstat.rootstats[SCAN_BY_SYSTEM][i]);
4747 }
4748 return 0;
4749}
4750
4751static int mem_cgroup_reset_vmscan_stat(struct cgroup *cgrp,
4752 unsigned int event)
4753{
4754 struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4755
4756 spin_lock(&mem->scanstat.lock);
4757 memset(&mem->scanstat.stats, 0, sizeof(mem->scanstat.stats));
4758 memset(&mem->scanstat.rootstats, 0, sizeof(mem->scanstat.rootstats));
4759 spin_unlock(&mem->scanstat.lock);
4760 return 0;
4761}
4762
4763
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004764static struct cftype mem_cgroup_files[] = {
4765 {
Balbir Singh0eea1032008-02-07 00:13:57 -08004766 .name = "usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004767 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
Paul Menage2c3daa72008-04-29 00:59:58 -07004768 .read_u64 = mem_cgroup_read,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004769 .register_event = mem_cgroup_usage_register_event,
4770 .unregister_event = mem_cgroup_usage_unregister_event,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004771 },
4772 {
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004773 .name = "max_usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004774 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07004775 .trigger = mem_cgroup_reset,
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07004776 .read_u64 = mem_cgroup_read,
4777 },
4778 {
Balbir Singh0eea1032008-02-07 00:13:57 -08004779 .name = "limit_in_bytes",
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004780 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
Paul Menage856c13a2008-07-25 01:47:04 -07004781 .write_string = mem_cgroup_write,
Paul Menage2c3daa72008-04-29 00:59:58 -07004782 .read_u64 = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004783 },
4784 {
Balbir Singh296c81d2009-09-23 15:56:36 -07004785 .name = "soft_limit_in_bytes",
4786 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4787 .write_string = mem_cgroup_write,
4788 .read_u64 = mem_cgroup_read,
4789 },
4790 {
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004791 .name = "failcnt",
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004792 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07004793 .trigger = mem_cgroup_reset,
Paul Menage2c3daa72008-04-29 00:59:58 -07004794 .read_u64 = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004795 },
Balbir Singh8697d332008-02-07 00:13:59 -08004796 {
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004797 .name = "stat",
Paul Menagec64745c2008-04-29 01:00:02 -07004798 .read_map = mem_control_stat_show,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08004799 },
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004800 {
4801 .name = "force_empty",
4802 .trigger = mem_cgroup_force_empty_write,
4803 },
Balbir Singh18f59ea2009-01-07 18:08:07 -08004804 {
4805 .name = "use_hierarchy",
4806 .write_u64 = mem_cgroup_hierarchy_write,
4807 .read_u64 = mem_cgroup_hierarchy_read,
4808 },
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08004809 {
4810 .name = "swappiness",
4811 .read_u64 = mem_cgroup_swappiness_read,
4812 .write_u64 = mem_cgroup_swappiness_write,
4813 },
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08004814 {
4815 .name = "move_charge_at_immigrate",
4816 .read_u64 = mem_cgroup_move_charge_read,
4817 .write_u64 = mem_cgroup_move_charge_write,
4818 },
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004819 {
4820 .name = "oom_control",
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004821 .read_map = mem_cgroup_oom_control_read,
4822 .write_u64 = mem_cgroup_oom_control_write,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004823 .register_event = mem_cgroup_oom_register_event,
4824 .unregister_event = mem_cgroup_oom_unregister_event,
4825 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4826 },
Ying Han406eb0c2011-05-26 16:25:37 -07004827#ifdef CONFIG_NUMA
4828 {
4829 .name = "numa_stat",
4830 .open = mem_control_numa_stat_open,
KAMEZAWA Hiroyuki89577122011-06-15 15:08:41 -07004831 .mode = S_IRUGO,
Ying Han406eb0c2011-05-26 16:25:37 -07004832 },
4833#endif
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07004834 {
4835 .name = "vmscan_stat",
4836 .read_map = mem_cgroup_vmscan_stat_read,
4837 .trigger = mem_cgroup_reset_vmscan_stat,
4838 },
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004839};
4840
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004841#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4842static struct cftype memsw_cgroup_files[] = {
4843 {
4844 .name = "memsw.usage_in_bytes",
4845 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
4846 .read_u64 = mem_cgroup_read,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07004847 .register_event = mem_cgroup_usage_register_event,
4848 .unregister_event = mem_cgroup_usage_unregister_event,
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004849 },
4850 {
4851 .name = "memsw.max_usage_in_bytes",
4852 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
4853 .trigger = mem_cgroup_reset,
4854 .read_u64 = mem_cgroup_read,
4855 },
4856 {
4857 .name = "memsw.limit_in_bytes",
4858 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
4859 .write_string = mem_cgroup_write,
4860 .read_u64 = mem_cgroup_read,
4861 },
4862 {
4863 .name = "memsw.failcnt",
4864 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
4865 .trigger = mem_cgroup_reset,
4866 .read_u64 = mem_cgroup_read,
4867 },
4868};
4869
4870static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
4871{
4872 if (!do_swap_account)
4873 return 0;
4874 return cgroup_add_files(cont, ss, memsw_cgroup_files,
4875 ARRAY_SIZE(memsw_cgroup_files));
4876};
4877#else
4878static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
4879{
4880 return 0;
4881}
4882#endif
4883
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004884static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
4885{
4886 struct mem_cgroup_per_node *pn;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004887 struct mem_cgroup_per_zone *mz;
Christoph Lameterb69408e2008-10-18 20:26:14 -07004888 enum lru_list l;
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07004889 int zone, tmp = node;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004890 /*
4891 * This routine is called against possible nodes.
4892 * But it's BUG to call kmalloc() against offline node.
4893 *
4894 * TODO: this routine can waste much memory for nodes which will
4895 * never be onlined. It's better to use memory hotplug callback
4896 * function.
4897 */
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07004898 if (!node_state(node, N_NORMAL_MEMORY))
4899 tmp = -1;
Jesper Juhl17295c82011-01-13 15:47:42 -08004900 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004901 if (!pn)
4902 return 1;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004903
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004904 mem->info.nodeinfo[node] = pn;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004905 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4906 mz = &pn->zoneinfo[zone];
Christoph Lameterb69408e2008-10-18 20:26:14 -07004907 for_each_lru(l)
4908 INIT_LIST_HEAD(&mz->lists[l]);
Balbir Singhf64c3f52009-09-23 15:56:37 -07004909 mz->usage_in_excess = 0;
Balbir Singh4e416952009-09-23 15:56:39 -07004910 mz->on_tree = false;
4911 mz->mem = mem;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004912 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004913 return 0;
4914}
4915
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004916static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
4917{
4918 kfree(mem->info.nodeinfo[node]);
4919}
4920
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004921static struct mem_cgroup *mem_cgroup_alloc(void)
4922{
4923 struct mem_cgroup *mem;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004924 int size = sizeof(struct mem_cgroup);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004925
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004926 /* Can be very big if MAX_NUMNODES is very big */
Jan Blunckc8dad2b2009-01-07 18:07:53 -08004927 if (size < PAGE_SIZE)
Jesper Juhl17295c82011-01-13 15:47:42 -08004928 mem = kzalloc(size, GFP_KERNEL);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004929 else
Jesper Juhl17295c82011-01-13 15:47:42 -08004930 mem = vzalloc(size);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004931
Dan Carpentere7bbcdf2010-03-23 13:35:12 -07004932 if (!mem)
4933 return NULL;
4934
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004935 mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
Dan Carpenterd2e61b82010-11-11 14:05:12 -08004936 if (!mem->stat)
4937 goto out_free;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07004938 spin_lock_init(&mem->pcp_counter_lock);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004939 return mem;
Dan Carpenterd2e61b82010-11-11 14:05:12 -08004940
4941out_free:
4942 if (size < PAGE_SIZE)
4943 kfree(mem);
4944 else
4945 vfree(mem);
4946 return NULL;
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004947}
4948
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004949/*
4950 * At destroying mem_cgroup, references from swap_cgroup can remain.
4951 * (scanning all at force_empty is too costly...)
4952 *
4953 * Instead of clearing all references at force_empty, we remember
4954 * the number of reference from swap_cgroup and free mem_cgroup when
4955 * it goes down to 0.
4956 *
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004957 * Removal of cgroup itself succeeds regardless of refs from swap.
4958 */
4959
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004960static void __mem_cgroup_free(struct mem_cgroup *mem)
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004961{
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004962 int node;
4963
Balbir Singhf64c3f52009-09-23 15:56:37 -07004964 mem_cgroup_remove_from_trees(mem);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004965 free_css_id(&mem_cgroup_subsys, &mem->css);
4966
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004967 for_each_node_state(node, N_POSSIBLE)
4968 free_mem_cgroup_per_zone_info(mem, node);
4969
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08004970 free_percpu(mem->stat);
4971 if (sizeof(struct mem_cgroup) < PAGE_SIZE)
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07004972 kfree(mem);
4973 else
4974 vfree(mem);
4975}
4976
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004977static void mem_cgroup_get(struct mem_cgroup *mem)
4978{
4979 atomic_inc(&mem->refcnt);
4980}
4981
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004982static void __mem_cgroup_put(struct mem_cgroup *mem, int count)
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004983{
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004984 if (atomic_sub_and_test(count, &mem->refcnt)) {
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004985 struct mem_cgroup *parent = parent_mem_cgroup(mem);
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08004986 __mem_cgroup_free(mem);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004987 if (parent)
4988 mem_cgroup_put(parent);
4989 }
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08004990}
4991
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004992static void mem_cgroup_put(struct mem_cgroup *mem)
4993{
4994 __mem_cgroup_put(mem, 1);
4995}
4996
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08004997/*
4998 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
4999 */
5000static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
5001{
5002 if (!mem->res.parent)
5003 return NULL;
5004 return mem_cgroup_from_res_counter(mem->res.parent, res);
5005}
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07005006
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08005007#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
5008static void __init enable_swap_cgroup(void)
5009{
Hirokazu Takahashif8d66542009-01-07 18:08:02 -08005010 if (!mem_cgroup_disabled() && really_do_swap_account)
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08005011 do_swap_account = 1;
5012}
5013#else
5014static void __init enable_swap_cgroup(void)
5015{
5016}
5017#endif
5018
Balbir Singhf64c3f52009-09-23 15:56:37 -07005019static int mem_cgroup_soft_limit_tree_init(void)
5020{
5021 struct mem_cgroup_tree_per_node *rtpn;
5022 struct mem_cgroup_tree_per_zone *rtpz;
5023 int tmp, node, zone;
5024
5025 for_each_node_state(node, N_POSSIBLE) {
5026 tmp = node;
5027 if (!node_state(node, N_NORMAL_MEMORY))
5028 tmp = -1;
5029 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
5030 if (!rtpn)
5031 return 1;
5032
5033 soft_limit_tree.rb_tree_per_node[node] = rtpn;
5034
5035 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
5036 rtpz = &rtpn->rb_tree_per_zone[zone];
5037 rtpz->rb_root = RB_ROOT;
5038 spin_lock_init(&rtpz->lock);
5039 }
5040 }
5041 return 0;
5042}
5043
Li Zefan0eb253e2009-01-15 13:51:25 -08005044static struct cgroup_subsys_state * __ref
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005045mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
5046{
Balbir Singh28dbc4b2009-01-07 18:08:05 -08005047 struct mem_cgroup *mem, *parent;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07005048 long error = -ENOMEM;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005049 int node;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005050
Jan Blunckc8dad2b2009-01-07 18:07:53 -08005051 mem = mem_cgroup_alloc();
5052 if (!mem)
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07005053 return ERR_PTR(error);
Pavel Emelianov78fb7462008-02-07 00:13:51 -08005054
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005055 for_each_node_state(node, N_POSSIBLE)
5056 if (alloc_mem_cgroup_per_zone_info(mem, node))
5057 goto free_out;
Balbir Singhf64c3f52009-09-23 15:56:37 -07005058
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08005059 /* root ? */
Balbir Singh28dbc4b2009-01-07 18:08:05 -08005060 if (cont->parent == NULL) {
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08005061 int cpu;
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08005062 enable_swap_cgroup();
Balbir Singh28dbc4b2009-01-07 18:08:05 -08005063 parent = NULL;
Balbir Singh4b3bde42009-09-23 15:56:32 -07005064 root_mem_cgroup = mem;
Balbir Singhf64c3f52009-09-23 15:56:37 -07005065 if (mem_cgroup_soft_limit_tree_init())
5066 goto free_out;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08005067 for_each_possible_cpu(cpu) {
5068 struct memcg_stock_pcp *stock =
5069 &per_cpu(memcg_stock, cpu);
5070 INIT_WORK(&stock->work, drain_local_stock);
5071 }
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07005072 hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
Balbir Singh18f59ea2009-01-07 18:08:07 -08005073 } else {
Balbir Singh28dbc4b2009-01-07 18:08:05 -08005074 parent = mem_cgroup_from_cont(cont->parent);
Balbir Singh18f59ea2009-01-07 18:08:07 -08005075 mem->use_hierarchy = parent->use_hierarchy;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005076 mem->oom_kill_disable = parent->oom_kill_disable;
Balbir Singh18f59ea2009-01-07 18:08:07 -08005077 }
Balbir Singh28dbc4b2009-01-07 18:08:05 -08005078
Balbir Singh18f59ea2009-01-07 18:08:07 -08005079 if (parent && parent->use_hierarchy) {
5080 res_counter_init(&mem->res, &parent->res);
5081 res_counter_init(&mem->memsw, &parent->memsw);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08005082 /*
5083 * We increment refcnt of the parent to ensure that we can
5084 * safely access it on res_counter_charge/uncharge.
5085 * This refcnt will be decremented when freeing this
5086 * mem_cgroup(see mem_cgroup_put).
5087 */
5088 mem_cgroup_get(parent);
Balbir Singh18f59ea2009-01-07 18:08:07 -08005089 } else {
5090 res_counter_init(&mem->res, NULL);
5091 res_counter_init(&mem->memsw, NULL);
5092 }
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07005093 mem->last_scanned_child = 0;
Ying Han889976d2011-05-26 16:25:33 -07005094 mem->last_scanned_node = MAX_NUMNODES;
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005095 INIT_LIST_HEAD(&mem->oom_notify);
Balbir Singh6d61ef42009-01-07 18:08:06 -08005096
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005097 if (parent)
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07005098 mem->swappiness = mem_cgroup_swappiness(parent);
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08005099 atomic_set(&mem->refcnt, 1);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005100 mem->move_charge_at_immigrate = 0;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005101 mutex_init(&mem->thresholds_lock);
KAMEZAWA Hiroyuki82f9d482011-07-26 16:08:26 -07005102 spin_lock_init(&mem->scanstat.lock);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005103 return &mem->css;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005104free_out:
KAMEZAWA Hiroyukia7ba0ee2009-01-07 18:08:32 -08005105 __mem_cgroup_free(mem);
Balbir Singh4b3bde42009-09-23 15:56:32 -07005106 root_mem_cgroup = NULL;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07005107 return ERR_PTR(error);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005108}
5109
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07005110static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08005111 struct cgroup *cont)
5112{
5113 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07005114
5115 return mem_cgroup_force_empty(mem, false);
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08005116}
5117
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005118static void mem_cgroup_destroy(struct cgroup_subsys *ss,
5119 struct cgroup *cont)
5120{
Daisuke Nishimurac268e992009-01-15 13:51:13 -08005121 struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
Daisuke Nishimurac268e992009-01-15 13:51:13 -08005122
Daisuke Nishimurac268e992009-01-15 13:51:13 -08005123 mem_cgroup_put(mem);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005124}
5125
5126static int mem_cgroup_populate(struct cgroup_subsys *ss,
5127 struct cgroup *cont)
5128{
KAMEZAWA Hiroyuki8c7c6e32009-01-07 18:08:00 -08005129 int ret;
5130
5131 ret = cgroup_add_files(cont, ss, mem_cgroup_files,
5132 ARRAY_SIZE(mem_cgroup_files));
5133
5134 if (!ret)
5135 ret = register_memsw_files(cont, ss);
5136 return ret;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005137}
5138
Daisuke Nishimura02491442010-03-10 15:22:17 -08005139#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005140/* Handlers for move charge at task migration. */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005141#define PRECHARGE_COUNT_AT_ONCE 256
5142static int mem_cgroup_do_precharge(unsigned long count)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005143{
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005144 int ret = 0;
5145 int batch_count = PRECHARGE_COUNT_AT_ONCE;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005146 struct mem_cgroup *mem = mc.to;
5147
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005148 if (mem_cgroup_is_root(mem)) {
5149 mc.precharge += count;
5150 /* we don't need css_get for root */
5151 return ret;
5152 }
5153 /* try to charge at once */
5154 if (count > 1) {
5155 struct res_counter *dummy;
5156 /*
5157 * "mem" cannot be under rmdir() because we've already checked
5158 * by cgroup_lock_live_cgroup() that it is not removed and we
5159 * are still under the same cgroup_mutex. So we can postpone
5160 * css_get().
5161 */
5162 if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
5163 goto one_by_one;
5164 if (do_swap_account && res_counter_charge(&mem->memsw,
5165 PAGE_SIZE * count, &dummy)) {
5166 res_counter_uncharge(&mem->res, PAGE_SIZE * count);
5167 goto one_by_one;
5168 }
5169 mc.precharge += count;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005170 return ret;
5171 }
5172one_by_one:
5173 /* fall back to one by one charge */
5174 while (count--) {
5175 if (signal_pending(current)) {
5176 ret = -EINTR;
5177 break;
5178 }
5179 if (!batch_count--) {
5180 batch_count = PRECHARGE_COUNT_AT_ONCE;
5181 cond_resched();
5182 }
Johannes Weiner7ec99d62011-03-23 16:42:36 -07005183 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, 1, &mem, false);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005184 if (ret || !mem)
5185 /* mem_cgroup_clear_mc() will do uncharge later */
5186 return -ENOMEM;
5187 mc.precharge++;
5188 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005189 return ret;
5190}
5191
5192/**
5193 * is_target_pte_for_mc - check a pte whether it is valid for move charge
5194 * @vma: the vma the pte to be checked belongs
5195 * @addr: the address corresponding to the pte to be checked
5196 * @ptent: the pte to be checked
Daisuke Nishimura02491442010-03-10 15:22:17 -08005197 * @target: the pointer the target page or swap ent will be stored(can be NULL)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005198 *
5199 * Returns
5200 * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
5201 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
5202 * move charge. if @target is not NULL, the page is stored in target->page
5203 * with extra refcnt got(Callers should handle it).
Daisuke Nishimura02491442010-03-10 15:22:17 -08005204 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
5205 * target for charge migration. if @target is not NULL, the entry is stored
5206 * in target->ent.
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005207 *
5208 * Called with pte lock held.
5209 */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005210union mc_target {
5211 struct page *page;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005212 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005213};
5214
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005215enum mc_target_type {
5216 MC_TARGET_NONE, /* not used */
5217 MC_TARGET_PAGE,
Daisuke Nishimura02491442010-03-10 15:22:17 -08005218 MC_TARGET_SWAP,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005219};
5220
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005221static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
5222 unsigned long addr, pte_t ptent)
5223{
5224 struct page *page = vm_normal_page(vma, addr, ptent);
5225
5226 if (!page || !page_mapped(page))
5227 return NULL;
5228 if (PageAnon(page)) {
5229 /* we don't move shared anon */
5230 if (!move_anon() || page_mapcount(page) > 2)
5231 return NULL;
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005232 } else if (!move_file())
5233 /* we ignore mapcount for file pages */
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005234 return NULL;
5235 if (!get_page_unless_zero(page))
5236 return NULL;
5237
5238 return page;
5239}
5240
5241static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
5242 unsigned long addr, pte_t ptent, swp_entry_t *entry)
5243{
5244 int usage_count;
5245 struct page *page = NULL;
5246 swp_entry_t ent = pte_to_swp_entry(ptent);
5247
5248 if (!move_anon() || non_swap_entry(ent))
5249 return NULL;
5250 usage_count = mem_cgroup_count_swap_user(ent, &page);
5251 if (usage_count > 1) { /* we don't move shared anon */
5252 if (page)
5253 put_page(page);
5254 return NULL;
5255 }
5256 if (do_swap_account)
5257 entry->val = ent.val;
5258
5259 return page;
5260}
5261
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005262static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
5263 unsigned long addr, pte_t ptent, swp_entry_t *entry)
5264{
5265 struct page *page = NULL;
5266 struct inode *inode;
5267 struct address_space *mapping;
5268 pgoff_t pgoff;
5269
5270 if (!vma->vm_file) /* anonymous vma */
5271 return NULL;
5272 if (!move_file())
5273 return NULL;
5274
5275 inode = vma->vm_file->f_path.dentry->d_inode;
5276 mapping = vma->vm_file->f_mapping;
5277 if (pte_none(ptent))
5278 pgoff = linear_page_index(vma, addr);
5279 else /* pte_file(ptent) is true */
5280 pgoff = pte_to_pgoff(ptent);
5281
5282 /* page is moved even if it's not RSS of this task(page-faulted). */
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07005283 page = find_get_page(mapping, pgoff);
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005284
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07005285#ifdef CONFIG_SWAP
5286 /* shmem/tmpfs may report page out on swap: account for that too. */
5287 if (radix_tree_exceptional_entry(page)) {
5288 swp_entry_t swap = radix_to_swp_entry(page);
5289 if (do_swap_account)
5290 *entry = swap;
5291 page = find_get_page(&swapper_space, swap.val);
5292 }
5293#endif
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005294 return page;
5295}
5296
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005297static int is_target_pte_for_mc(struct vm_area_struct *vma,
5298 unsigned long addr, pte_t ptent, union mc_target *target)
5299{
Daisuke Nishimura02491442010-03-10 15:22:17 -08005300 struct page *page = NULL;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005301 struct page_cgroup *pc;
5302 int ret = 0;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005303 swp_entry_t ent = { .val = 0 };
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005304
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005305 if (pte_present(ptent))
5306 page = mc_handle_present_pte(vma, addr, ptent);
5307 else if (is_swap_pte(ptent))
5308 page = mc_handle_swap_pte(vma, addr, ptent, &ent);
Daisuke Nishimura87946a72010-05-26 14:42:39 -07005309 else if (pte_none(ptent) || pte_file(ptent))
5310 page = mc_handle_file_pte(vma, addr, ptent, &ent);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005311
5312 if (!page && !ent.val)
Daisuke Nishimura02491442010-03-10 15:22:17 -08005313 return 0;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005314 if (page) {
5315 pc = lookup_page_cgroup(page);
5316 /*
5317 * Do only loose check w/o page_cgroup lock.
5318 * mem_cgroup_move_account() checks the pc is valid or not under
5319 * the lock.
5320 */
5321 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
5322 ret = MC_TARGET_PAGE;
5323 if (target)
5324 target->page = page;
5325 }
5326 if (!ret || !target)
5327 put_page(page);
5328 }
Daisuke Nishimura90254a62010-05-26 14:42:38 -07005329 /* There is a swap entry and a page doesn't exist or isn't charged */
5330 if (ent.val && !ret &&
KAMEZAWA Hiroyuki7f0f1542010-05-11 14:06:58 -07005331 css_id(&mc.from->css) == lookup_swap_cgroup(ent)) {
5332 ret = MC_TARGET_SWAP;
5333 if (target)
5334 target->ent = ent;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005335 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005336 return ret;
5337}
5338
5339static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
5340 unsigned long addr, unsigned long end,
5341 struct mm_walk *walk)
5342{
5343 struct vm_area_struct *vma = walk->private;
5344 pte_t *pte;
5345 spinlock_t *ptl;
5346
Dave Hansen03319322011-03-22 16:32:56 -07005347 split_huge_page_pmd(walk->mm, pmd);
5348
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005349 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
5350 for (; addr != end; pte++, addr += PAGE_SIZE)
5351 if (is_target_pte_for_mc(vma, addr, *pte, NULL))
5352 mc.precharge++; /* increment precharge temporarily */
5353 pte_unmap_unlock(pte - 1, ptl);
5354 cond_resched();
5355
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005356 return 0;
5357}
5358
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005359static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
5360{
5361 unsigned long precharge;
5362 struct vm_area_struct *vma;
5363
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005364 down_read(&mm->mmap_sem);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005365 for (vma = mm->mmap; vma; vma = vma->vm_next) {
5366 struct mm_walk mem_cgroup_count_precharge_walk = {
5367 .pmd_entry = mem_cgroup_count_precharge_pte_range,
5368 .mm = mm,
5369 .private = vma,
5370 };
5371 if (is_vm_hugetlb_page(vma))
5372 continue;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005373 walk_page_range(vma->vm_start, vma->vm_end,
5374 &mem_cgroup_count_precharge_walk);
5375 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005376 up_read(&mm->mmap_sem);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005377
5378 precharge = mc.precharge;
5379 mc.precharge = 0;
5380
5381 return precharge;
5382}
5383
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005384static int mem_cgroup_precharge_mc(struct mm_struct *mm)
5385{
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005386 unsigned long precharge = mem_cgroup_count_precharge(mm);
5387
5388 VM_BUG_ON(mc.moving_task);
5389 mc.moving_task = current;
5390 return mem_cgroup_do_precharge(precharge);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005391}
5392
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005393/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
5394static void __mem_cgroup_clear_mc(void)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005395{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005396 struct mem_cgroup *from = mc.from;
5397 struct mem_cgroup *to = mc.to;
5398
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005399 /* we must uncharge all the leftover precharges from mc.to */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005400 if (mc.precharge) {
5401 __mem_cgroup_cancel_charge(mc.to, mc.precharge);
5402 mc.precharge = 0;
5403 }
5404 /*
5405 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
5406 * we must uncharge here.
5407 */
5408 if (mc.moved_charge) {
5409 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
5410 mc.moved_charge = 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005411 }
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005412 /* we must fixup refcnts and charges */
5413 if (mc.moved_swap) {
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005414 /* uncharge swap account from the old cgroup */
5415 if (!mem_cgroup_is_root(mc.from))
5416 res_counter_uncharge(&mc.from->memsw,
5417 PAGE_SIZE * mc.moved_swap);
5418 __mem_cgroup_put(mc.from, mc.moved_swap);
5419
5420 if (!mem_cgroup_is_root(mc.to)) {
5421 /*
5422 * we charged both to->res and to->memsw, so we should
5423 * uncharge to->res.
5424 */
5425 res_counter_uncharge(&mc.to->res,
5426 PAGE_SIZE * mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005427 }
5428 /* we've already done mem_cgroup_get(mc.to) */
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005429 mc.moved_swap = 0;
5430 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005431 memcg_oom_recover(from);
5432 memcg_oom_recover(to);
5433 wake_up_all(&mc.waitq);
5434}
5435
5436static void mem_cgroup_clear_mc(void)
5437{
5438 struct mem_cgroup *from = mc.from;
5439
5440 /*
5441 * we must clear moving_task before waking up waiters at the end of
5442 * task migration.
5443 */
5444 mc.moving_task = NULL;
5445 __mem_cgroup_clear_mc();
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005446 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005447 mc.from = NULL;
5448 mc.to = NULL;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005449 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07005450 mem_cgroup_end_move(from);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005451}
5452
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005453static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
5454 struct cgroup *cgroup,
Ben Blumf780bdb2011-05-26 16:25:19 -07005455 struct task_struct *p)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005456{
5457 int ret = 0;
5458 struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup);
5459
5460 if (mem->move_charge_at_immigrate) {
5461 struct mm_struct *mm;
5462 struct mem_cgroup *from = mem_cgroup_from_task(p);
5463
5464 VM_BUG_ON(from == mem);
5465
5466 mm = get_task_mm(p);
5467 if (!mm)
5468 return 0;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005469 /* We move charges only when we move a owner of the mm */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005470 if (mm->owner == p) {
5471 VM_BUG_ON(mc.from);
5472 VM_BUG_ON(mc.to);
5473 VM_BUG_ON(mc.precharge);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005474 VM_BUG_ON(mc.moved_charge);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005475 VM_BUG_ON(mc.moved_swap);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07005476 mem_cgroup_start_move(from);
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005477 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005478 mc.from = from;
5479 mc.to = mem;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07005480 spin_unlock(&mc.lock);
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005481 /* We set mc.moving_task later */
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005482
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005483 ret = mem_cgroup_precharge_mc(mm);
5484 if (ret)
5485 mem_cgroup_clear_mc();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005486 }
5487 mmput(mm);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005488 }
5489 return ret;
5490}
5491
5492static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
5493 struct cgroup *cgroup,
Ben Blumf780bdb2011-05-26 16:25:19 -07005494 struct task_struct *p)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005495{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005496 mem_cgroup_clear_mc();
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005497}
5498
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005499static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
5500 unsigned long addr, unsigned long end,
5501 struct mm_walk *walk)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005502{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005503 int ret = 0;
5504 struct vm_area_struct *vma = walk->private;
5505 pte_t *pte;
5506 spinlock_t *ptl;
5507
Dave Hansen03319322011-03-22 16:32:56 -07005508 split_huge_page_pmd(walk->mm, pmd);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005509retry:
5510 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
5511 for (; addr != end; addr += PAGE_SIZE) {
5512 pte_t ptent = *(pte++);
5513 union mc_target target;
5514 int type;
5515 struct page *page;
5516 struct page_cgroup *pc;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005517 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005518
5519 if (!mc.precharge)
5520 break;
5521
5522 type = is_target_pte_for_mc(vma, addr, ptent, &target);
5523 switch (type) {
5524 case MC_TARGET_PAGE:
5525 page = target.page;
5526 if (isolate_lru_page(page))
5527 goto put;
5528 pc = lookup_page_cgroup(page);
Johannes Weiner7ec99d62011-03-23 16:42:36 -07005529 if (!mem_cgroup_move_account(page, 1, pc,
5530 mc.from, mc.to, false)) {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005531 mc.precharge--;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005532 /* we uncharge from mc.from later. */
5533 mc.moved_charge++;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005534 }
5535 putback_lru_page(page);
5536put: /* is_target_pte_for_mc() gets the page */
5537 put_page(page);
5538 break;
Daisuke Nishimura02491442010-03-10 15:22:17 -08005539 case MC_TARGET_SWAP:
5540 ent = target.ent;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005541 if (!mem_cgroup_move_swap_account(ent,
5542 mc.from, mc.to, false)) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08005543 mc.precharge--;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08005544 /* we fixup refcnts and charges later. */
5545 mc.moved_swap++;
5546 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08005547 break;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005548 default:
5549 break;
5550 }
5551 }
5552 pte_unmap_unlock(pte - 1, ptl);
5553 cond_resched();
5554
5555 if (addr != end) {
5556 /*
5557 * We have consumed all precharges we got in can_attach().
5558 * We try charge one by one, but don't do any additional
5559 * charges to mc.to if we have failed in charge once in attach()
5560 * phase.
5561 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08005562 ret = mem_cgroup_do_precharge(1);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005563 if (!ret)
5564 goto retry;
5565 }
5566
5567 return ret;
5568}
5569
5570static void mem_cgroup_move_charge(struct mm_struct *mm)
5571{
5572 struct vm_area_struct *vma;
5573
5574 lru_add_drain_all();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005575retry:
5576 if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
5577 /*
5578 * Someone who are holding the mmap_sem might be waiting in
5579 * waitq. So we cancel all extra charges, wake up all waiters,
5580 * and retry. Because we cancel precharges, we might not be able
5581 * to move enough charges, but moving charge is a best-effort
5582 * feature anyway, so it wouldn't be a big problem.
5583 */
5584 __mem_cgroup_clear_mc();
5585 cond_resched();
5586 goto retry;
5587 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005588 for (vma = mm->mmap; vma; vma = vma->vm_next) {
5589 int ret;
5590 struct mm_walk mem_cgroup_move_charge_walk = {
5591 .pmd_entry = mem_cgroup_move_charge_pte_range,
5592 .mm = mm,
5593 .private = vma,
5594 };
5595 if (is_vm_hugetlb_page(vma))
5596 continue;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08005597 ret = walk_page_range(vma->vm_start, vma->vm_end,
5598 &mem_cgroup_move_charge_walk);
5599 if (ret)
5600 /*
5601 * means we have consumed all precharges and failed in
5602 * doing additional charge. Just abandon here.
5603 */
5604 break;
5605 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005606 up_read(&mm->mmap_sem);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005607}
5608
Balbir Singh67e465a2008-02-07 00:13:54 -08005609static void mem_cgroup_move_task(struct cgroup_subsys *ss,
5610 struct cgroup *cont,
5611 struct cgroup *old_cont,
Ben Blumf780bdb2011-05-26 16:25:19 -07005612 struct task_struct *p)
Balbir Singh67e465a2008-02-07 00:13:54 -08005613{
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07005614 struct mm_struct *mm = get_task_mm(p);
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005615
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005616 if (mm) {
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07005617 if (mc.to)
5618 mem_cgroup_move_charge(mm);
5619 put_swap_token(mm);
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08005620 mmput(mm);
5621 }
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07005622 if (mc.to)
5623 mem_cgroup_clear_mc();
Balbir Singh67e465a2008-02-07 00:13:54 -08005624}
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07005625#else /* !CONFIG_MMU */
5626static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
5627 struct cgroup *cgroup,
Ben Blumf780bdb2011-05-26 16:25:19 -07005628 struct task_struct *p)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07005629{
5630 return 0;
5631}
5632static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
5633 struct cgroup *cgroup,
Ben Blumf780bdb2011-05-26 16:25:19 -07005634 struct task_struct *p)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07005635{
5636}
5637static void mem_cgroup_move_task(struct cgroup_subsys *ss,
5638 struct cgroup *cont,
5639 struct cgroup *old_cont,
Ben Blumf780bdb2011-05-26 16:25:19 -07005640 struct task_struct *p)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07005641{
5642}
5643#endif
Balbir Singh67e465a2008-02-07 00:13:54 -08005644
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005645struct cgroup_subsys mem_cgroup_subsys = {
5646 .name = "memory",
5647 .subsys_id = mem_cgroup_subsys_id,
5648 .create = mem_cgroup_create,
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08005649 .pre_destroy = mem_cgroup_pre_destroy,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005650 .destroy = mem_cgroup_destroy,
5651 .populate = mem_cgroup_populate,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005652 .can_attach = mem_cgroup_can_attach,
5653 .cancel_attach = mem_cgroup_cancel_attach,
Balbir Singh67e465a2008-02-07 00:13:54 -08005654 .attach = mem_cgroup_move_task,
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005655 .early_init = 0,
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07005656 .use_id = 1,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005657};
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08005658
5659#ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
Michal Hockoa42c3902010-11-24 12:57:08 -08005660static int __init enable_swap_account(char *s)
5661{
5662 /* consider enabled if no parameter or 1 is given */
Michal Hockoa2c89902011-05-24 17:12:50 -07005663 if (!strcmp(s, "1"))
Michal Hockoa42c3902010-11-24 12:57:08 -08005664 really_do_swap_account = 1;
Michal Hockoa2c89902011-05-24 17:12:50 -07005665 else if (!strcmp(s, "0"))
Michal Hockoa42c3902010-11-24 12:57:08 -08005666 really_do_swap_account = 0;
5667 return 1;
5668}
Michal Hockoa2c89902011-05-24 17:12:50 -07005669__setup("swapaccount=", enable_swap_account);
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08005670
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08005671#endif