blob: 09255ec8159c459624e66ee5ac17ca092d2fcd6b [file] [log] [blame]
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001/* memcontrol.c - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08006 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08009 * Memory thresholds
10 * Copyright (C) 2009 Nokia Corporation
11 * Author: Kirill A. Shutemov
12 *
Glauber Costa7ae1e1d2012-12-18 14:21:56 -080013 * Kernel Memory Controller
14 * Copyright (C) 2012 Parallels Inc. and Google Inc.
15 * Authors: Glauber Costa and Suleiman Souhlal
16 *
Balbir Singh8cdea7c2008-02-07 00:13:50 -080017 * This program is free software; you can redistribute it and/or modify
18 * it under the terms of the GNU General Public License as published by
19 * the Free Software Foundation; either version 2 of the License, or
20 * (at your option) any later version.
21 *
22 * This program is distributed in the hope that it will be useful,
23 * but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU General Public License for more details.
26 */
27
28#include <linux/res_counter.h>
29#include <linux/memcontrol.h>
30#include <linux/cgroup.h>
Pavel Emelianov78fb7462008-02-07 00:13:51 -080031#include <linux/mm.h>
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -080032#include <linux/hugetlb.h>
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -080033#include <linux/pagemap.h>
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080034#include <linux/smp.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080035#include <linux/page-flags.h>
Balbir Singh66e17072008-02-07 00:13:56 -080036#include <linux/backing-dev.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080037#include <linux/bit_spinlock.h>
38#include <linux/rcupdate.h>
Balbir Singhe2224322009-04-02 16:57:39 -070039#include <linux/limits.h>
Paul Gortmakerb9e15ba2011-05-26 16:00:52 -040040#include <linux/export.h>
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -080041#include <linux/mutex.h>
Balbir Singhf64c3f52009-09-23 15:56:37 -070042#include <linux/rbtree.h>
Balbir Singhb6ac57d2008-04-29 01:00:19 -070043#include <linux/slab.h>
Balbir Singh66e17072008-02-07 00:13:56 -080044#include <linux/swap.h>
Daisuke Nishimura02491442010-03-10 15:22:17 -080045#include <linux/swapops.h>
Balbir Singh66e17072008-02-07 00:13:56 -080046#include <linux/spinlock.h>
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080047#include <linux/eventfd.h>
48#include <linux/sort.h>
Balbir Singh66e17072008-02-07 00:13:56 -080049#include <linux/fs.h>
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -080050#include <linux/seq_file.h>
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -070051#include <linux/vmalloc.h>
Christoph Lameterb69408e2008-10-18 20:26:14 -070052#include <linux/mm_inline.h>
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -070053#include <linux/page_cgroup.h>
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -080054#include <linux/cpu.h>
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -070055#include <linux/oom.h>
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -080056#include "internal.h"
Glauber Costad1a4c0b2011-12-11 21:47:04 +000057#include <net/sock.h>
Michal Hocko4bd2c1e2012-10-08 16:33:10 -070058#include <net/ip.h>
Glauber Costad1a4c0b2011-12-11 21:47:04 +000059#include <net/tcp_memcontrol.h>
Balbir Singh8cdea7c2008-02-07 00:13:50 -080060
Balbir Singh8697d332008-02-07 00:13:59 -080061#include <asm/uaccess.h>
62
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -070063#include <trace/events/vmscan.h>
64
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070065struct cgroup_subsys mem_cgroup_subsys __read_mostly;
David Rientjes68ae5642012-12-12 13:51:57 -080066EXPORT_SYMBOL(mem_cgroup_subsys);
67
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070068#define MEM_CGROUP_RECLAIM_RETRIES 5
Kirill A. Shutemov6bbda352012-05-29 15:06:55 -070069static struct mem_cgroup *root_mem_cgroup __read_mostly;
Balbir Singh8cdea7c2008-02-07 00:13:50 -080070
Andrew Mortonc255a452012-07-31 16:43:02 -070071#ifdef CONFIG_MEMCG_SWAP
Li Zefan338c8432009-06-17 16:27:15 -070072/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080073int do_swap_account __read_mostly;
Michal Hockoa42c3902010-11-24 12:57:08 -080074
75/* for remember boot option*/
Andrew Mortonc255a452012-07-31 16:43:02 -070076#ifdef CONFIG_MEMCG_SWAP_ENABLED
Michal Hockoa42c3902010-11-24 12:57:08 -080077static int really_do_swap_account __initdata = 1;
78#else
79static int really_do_swap_account __initdata = 0;
80#endif
81
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080082#else
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -070083#define do_swap_account 0
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080084#endif
85
86
Balbir Singh8cdea7c2008-02-07 00:13:50 -080087/*
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080088 * Statistics for memory cgroup.
89 */
90enum mem_cgroup_stat_index {
91 /*
92 * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
93 */
94 MEM_CGROUP_STAT_CACHE, /* # of pages charged as cache */
Balbir Singhd69b0422009-06-17 16:26:34 -070095 MEM_CGROUP_STAT_RSS, /* # of pages charged as anon rss */
KAMEZAWA Hiroyukid8046582009-12-15 16:47:09 -080096 MEM_CGROUP_STAT_FILE_MAPPED, /* # of pages charged as file rss */
Kamezawa Hiroyukibff6bb82012-07-31 16:41:38 -070097 MEM_CGROUP_STAT_SWAP, /* # of pages, swapped out */
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080098 MEM_CGROUP_STAT_NSTATS,
99};
100
Johannes Weineraf7c4b02012-05-29 15:07:08 -0700101static const char * const mem_cgroup_stat_names[] = {
102 "cache",
103 "rss",
104 "mapped_file",
105 "swap",
106};
107
Johannes Weinere9f89742011-03-23 16:42:37 -0700108enum mem_cgroup_events_index {
109 MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */
110 MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */
Ying Han456f9982011-05-26 16:25:38 -0700111 MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */
112 MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */
Johannes Weinere9f89742011-03-23 16:42:37 -0700113 MEM_CGROUP_EVENTS_NSTATS,
114};
Johannes Weineraf7c4b02012-05-29 15:07:08 -0700115
116static const char * const mem_cgroup_events_names[] = {
117 "pgpgin",
118 "pgpgout",
119 "pgfault",
120 "pgmajfault",
121};
122
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700123/*
124 * Per memcg event counter is incremented at every pagein/pageout. With THP,
125 * it will be incremated by the number of pages. This counter is used for
126 * for trigger some periodic events. This is straightforward and better
127 * than using jiffies etc. to handle periodic memcg event.
128 */
129enum mem_cgroup_events_target {
130 MEM_CGROUP_TARGET_THRESH,
131 MEM_CGROUP_TARGET_SOFTLIMIT,
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -0700132 MEM_CGROUP_TARGET_NUMAINFO,
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700133 MEM_CGROUP_NTARGETS,
134};
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700135#define THRESHOLDS_EVENTS_TARGET 128
136#define SOFTLIMIT_EVENTS_TARGET 1024
137#define NUMAINFO_EVENTS_TARGET 1024
Johannes Weinere9f89742011-03-23 16:42:37 -0700138
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800139struct mem_cgroup_stat_cpu {
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700140 long count[MEM_CGROUP_STAT_NSTATS];
Johannes Weinere9f89742011-03-23 16:42:37 -0700141 unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
Johannes Weiner13114712012-05-29 15:07:07 -0700142 unsigned long nr_page_events;
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700143 unsigned long targets[MEM_CGROUP_NTARGETS];
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800144};
145
Johannes Weiner527a5ec2012-01-12 17:17:55 -0800146struct mem_cgroup_reclaim_iter {
147 /* css_id of the last scanned hierarchy member */
148 int position;
149 /* scan generation, increased every round-trip */
150 unsigned int generation;
151};
152
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800153/*
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800154 * per-zone information in memory controller.
155 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800156struct mem_cgroup_per_zone {
Johannes Weiner6290df52012-01-12 17:18:10 -0800157 struct lruvec lruvec;
Hugh Dickins1eb49272012-03-21 16:34:19 -0700158 unsigned long lru_size[NR_LRU_LISTS];
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -0800159
Johannes Weiner527a5ec2012-01-12 17:17:55 -0800160 struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];
161
Balbir Singhf64c3f52009-09-23 15:56:37 -0700162 struct rb_node tree_node; /* RB tree node */
163 unsigned long long usage_in_excess;/* Set to the value by which */
164 /* the soft limit is exceeded*/
165 bool on_tree;
Hugh Dickinsd79154b2012-03-21 16:34:18 -0700166 struct mem_cgroup *memcg; /* Back pointer, we cannot */
Balbir Singh4e416952009-09-23 15:56:39 -0700167 /* use container_of */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800168};
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800169
170struct mem_cgroup_per_node {
171 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
172};
173
174struct mem_cgroup_lru_info {
175 struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
176};
177
178/*
Balbir Singhf64c3f52009-09-23 15:56:37 -0700179 * Cgroups above their limits are maintained in a RB-Tree, independent of
180 * their hierarchy representation
181 */
182
183struct mem_cgroup_tree_per_zone {
184 struct rb_root rb_root;
185 spinlock_t lock;
186};
187
188struct mem_cgroup_tree_per_node {
189 struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
190};
191
192struct mem_cgroup_tree {
193 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
194};
195
196static struct mem_cgroup_tree soft_limit_tree __read_mostly;
197
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800198struct mem_cgroup_threshold {
199 struct eventfd_ctx *eventfd;
200 u64 threshold;
201};
202
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700203/* For threshold */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800204struct mem_cgroup_threshold_ary {
Sha Zhengju748dad32012-05-29 15:06:57 -0700205 /* An array index points to threshold just below or equal to usage. */
Phil Carmody5407a562010-05-26 14:42:42 -0700206 int current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800207 /* Size of entries[] */
208 unsigned int size;
209 /* Array of thresholds */
210 struct mem_cgroup_threshold entries[0];
211};
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700212
213struct mem_cgroup_thresholds {
214 /* Primary thresholds array */
215 struct mem_cgroup_threshold_ary *primary;
216 /*
217 * Spare threshold array.
218 * This is needed to make mem_cgroup_unregister_event() "never fail".
219 * It must be able to store at least primary->size - 1 entries.
220 */
221 struct mem_cgroup_threshold_ary *spare;
222};
223
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700224/* for OOM */
225struct mem_cgroup_eventfd_list {
226 struct list_head list;
227 struct eventfd_ctx *eventfd;
228};
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800229
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700230static void mem_cgroup_threshold(struct mem_cgroup *memcg);
231static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800232
Balbir Singhf64c3f52009-09-23 15:56:37 -0700233/*
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800234 * The memory controller data structure. The memory controller controls both
235 * page cache and RSS per cgroup. We would eventually like to provide
236 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
237 * to help the administrator determine what knobs to tune.
238 *
239 * TODO: Add a water mark for the memory controller. Reclaim will begin when
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800240 * we hit the water mark. May be even add a low water mark, such that
241 * no reclaim occurs from a cgroup at it's low water mark, this is
242 * a feature that will be implemented much later in the future.
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800243 */
244struct mem_cgroup {
245 struct cgroup_subsys_state css;
246 /*
247 * the counter to account for memory usage
248 */
249 struct res_counter res;
Hugh Dickins59927fb2012-03-15 15:17:07 -0700250
251 union {
252 /*
253 * the counter to account for mem+swap usage.
254 */
255 struct res_counter memsw;
256
257 /*
258 * rcu_freeing is used only when freeing struct mem_cgroup,
259 * so put it into a union to avoid wasting more memory.
260 * It must be disjoint from the css field. It could be
261 * in a union with the res field, but res plays a much
262 * larger part in mem_cgroup life than memsw, and might
263 * be of interest, even at time of free, when debugging.
264 * So share rcu_head with the less interesting memsw.
265 */
266 struct rcu_head rcu_freeing;
267 /*
Glauber Costa3afe36b2012-05-29 15:07:10 -0700268 * We also need some space for a worker in deferred freeing.
269 * By the time we call it, rcu_freeing is no longer in use.
Hugh Dickins59927fb2012-03-15 15:17:07 -0700270 */
271 struct work_struct work_freeing;
272 };
273
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800274 /*
Glauber Costa510fc4e2012-12-18 14:21:47 -0800275 * the counter to account for kernel memory usage.
276 */
277 struct res_counter kmem;
278 /*
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800279 * Per cgroup active and inactive list, similar to the
280 * per zone LRU lists.
Pavel Emelianov78fb7462008-02-07 00:13:51 -0800281 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800282 struct mem_cgroup_lru_info info;
Ying Han889976d2011-05-26 16:25:33 -0700283 int last_scanned_node;
284#if MAX_NUMNODES > 1
285 nodemask_t scan_nodes;
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -0700286 atomic_t numainfo_events;
287 atomic_t numainfo_updating;
Ying Han889976d2011-05-26 16:25:33 -0700288#endif
Balbir Singh18f59ea2009-01-07 18:08:07 -0800289 /*
290 * Should the accounting and control be hierarchical, per subtree?
291 */
292 bool use_hierarchy;
Glauber Costa510fc4e2012-12-18 14:21:47 -0800293 unsigned long kmem_account_flags; /* See KMEM_ACCOUNTED_*, below */
Michal Hocko79dfdac2011-07-26 16:08:23 -0700294
295 bool oom_lock;
296 atomic_t under_oom;
297
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800298 atomic_t refcnt;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -0800299
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -0700300 int swappiness;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -0700301 /* OOM-Killer disable */
302 int oom_kill_disable;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -0800303
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -0700304 /* set when res.limit == memsw.limit */
305 bool memsw_is_minimum;
306
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800307 /* protect arrays of thresholds */
308 struct mutex thresholds_lock;
309
310 /* thresholds for memory usage. RCU-protected */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700311 struct mem_cgroup_thresholds thresholds;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -0700312
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800313 /* thresholds for mem+swap usage. RCU-protected */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700314 struct mem_cgroup_thresholds memsw_thresholds;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -0700315
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700316 /* For oom notifier event fd */
317 struct list_head oom_notify;
Johannes Weiner185efc02011-09-14 16:21:58 -0700318
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800319 /*
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800320 * Should we move charges of a task when a task is moved into this
321 * mem_cgroup ? And what type of charges should we move ?
322 */
323 unsigned long move_charge_at_immigrate;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800324 /*
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -0700325 * set > 0 if pages under this cgroup are moving to other cgroup.
326 */
327 atomic_t moving_account;
KAMEZAWA Hiroyuki312734c02012-03-21 16:34:24 -0700328 /* taken only while moving_account > 0 */
329 spinlock_t move_lock;
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -0700330 /*
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800331 * percpu counter.
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800332 */
Kirill A. Shutemov3a7951b2012-05-29 15:06:56 -0700333 struct mem_cgroup_stat_cpu __percpu *stat;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700334 /*
335 * used when a cpu is offlined or other synchronizations
336 * See mem_cgroup_read_stat().
337 */
338 struct mem_cgroup_stat_cpu nocpu_base;
339 spinlock_t pcp_counter_lock;
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000340
Michal Hocko4bd2c1e2012-10-08 16:33:10 -0700341#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000342 struct tcp_memcontrol tcp_mem;
343#endif
Glauber Costa2633d7a2012-12-18 14:22:34 -0800344#if defined(CONFIG_MEMCG_KMEM)
345 /* analogous to slab_common's slab_caches list. per-memcg */
346 struct list_head memcg_slab_caches;
347 /* Not a spinlock, we can take a lot of time walking the list */
348 struct mutex slab_caches_mutex;
349 /* Index in the kmem_cache->memcg_params->memcg_caches array */
350 int kmemcg_id;
351#endif
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800352};
353
Glauber Costa510fc4e2012-12-18 14:21:47 -0800354/* internal only representation about the status of kmem accounting. */
355enum {
356 KMEM_ACCOUNTED_ACTIVE = 0, /* accounted by this cgroup itself */
Glauber Costaa8964b92012-12-18 14:22:09 -0800357 KMEM_ACCOUNTED_ACTIVATED, /* static key enabled. */
Glauber Costa7de37682012-12-18 14:22:07 -0800358 KMEM_ACCOUNTED_DEAD, /* dead memcg with pending kmem charges */
Glauber Costa510fc4e2012-12-18 14:21:47 -0800359};
360
Glauber Costaa8964b92012-12-18 14:22:09 -0800361/* We account when limit is on, but only after call sites are patched */
362#define KMEM_ACCOUNTED_MASK \
363 ((1 << KMEM_ACCOUNTED_ACTIVE) | (1 << KMEM_ACCOUNTED_ACTIVATED))
Glauber Costa510fc4e2012-12-18 14:21:47 -0800364
365#ifdef CONFIG_MEMCG_KMEM
366static inline void memcg_kmem_set_active(struct mem_cgroup *memcg)
367{
368 set_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
369}
Glauber Costa7de37682012-12-18 14:22:07 -0800370
371static bool memcg_kmem_is_active(struct mem_cgroup *memcg)
372{
373 return test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
374}
375
Glauber Costaa8964b92012-12-18 14:22:09 -0800376static void memcg_kmem_set_activated(struct mem_cgroup *memcg)
377{
378 set_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
379}
380
Glauber Costa55007d82012-12-18 14:22:38 -0800381static void memcg_kmem_clear_activated(struct mem_cgroup *memcg)
382{
383 clear_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
384}
385
Glauber Costa7de37682012-12-18 14:22:07 -0800386static void memcg_kmem_mark_dead(struct mem_cgroup *memcg)
387{
388 if (test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags))
389 set_bit(KMEM_ACCOUNTED_DEAD, &memcg->kmem_account_flags);
390}
391
392static bool memcg_kmem_test_and_clear_dead(struct mem_cgroup *memcg)
393{
394 return test_and_clear_bit(KMEM_ACCOUNTED_DEAD,
395 &memcg->kmem_account_flags);
396}
Glauber Costa510fc4e2012-12-18 14:21:47 -0800397#endif
398
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800399/* Stuffs for move charges at task migration. */
400/*
401 * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
402 * left-shifted bitmap of these types.
403 */
404enum move_type {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800405 MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700406 MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800407 NR_MOVE_TYPE,
408};
409
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800410/* "mc" and its members are protected by cgroup_mutex */
411static struct move_charge_struct {
Daisuke Nishimurab1dd6932010-11-24 12:57:06 -0800412 spinlock_t lock; /* for from, to */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800413 struct mem_cgroup *from;
414 struct mem_cgroup *to;
415 unsigned long precharge;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -0800416 unsigned long moved_charge;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -0800417 unsigned long moved_swap;
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800418 struct task_struct *moving_task; /* a task moving charges */
419 wait_queue_head_t waitq; /* a waitq for other context */
420} mc = {
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -0700421 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800422 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
423};
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800424
Daisuke Nishimura90254a62010-05-26 14:42:38 -0700425static bool move_anon(void)
426{
427 return test_bit(MOVE_CHARGE_TYPE_ANON,
428 &mc.to->move_charge_at_immigrate);
429}
430
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700431static bool move_file(void)
432{
433 return test_bit(MOVE_CHARGE_TYPE_FILE,
434 &mc.to->move_charge_at_immigrate);
435}
436
Balbir Singh4e416952009-09-23 15:56:39 -0700437/*
438 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
439 * limit reclaim to prevent infinite loops, if they ever occur.
440 */
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700441#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
442#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
Balbir Singh4e416952009-09-23 15:56:39 -0700443
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800444enum charge_type {
445 MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -0700446 MEM_CGROUP_CHARGE_TYPE_ANON,
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -0800447 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -0700448 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700449 NR_CHARGE_TYPE,
450};
451
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800452/* for encoding cft->private value on file */
Glauber Costa86ae53e2012-12-18 14:21:45 -0800453enum res_type {
454 _MEM,
455 _MEMSWAP,
456 _OOM_TYPE,
Glauber Costa510fc4e2012-12-18 14:21:47 -0800457 _KMEM,
Glauber Costa86ae53e2012-12-18 14:21:45 -0800458};
459
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700460#define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
461#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800462#define MEMFILE_ATTR(val) ((val) & 0xffff)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700463/* Used for OOM nofiier */
464#define OOM_CONTROL (0)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800465
Balbir Singh75822b42009-09-23 15:56:38 -0700466/*
467 * Reclaim flags for mem_cgroup_hierarchical_reclaim
468 */
469#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0
470#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
471#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1
472#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
473
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700474static void mem_cgroup_get(struct mem_cgroup *memcg);
475static void mem_cgroup_put(struct mem_cgroup *memcg);
Glauber Costae1aab162011-12-11 21:47:03 +0000476
Wanpeng Lib2145142012-07-31 16:46:01 -0700477static inline
478struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s)
479{
480 return container_of(s, struct mem_cgroup, css);
481}
482
Michal Hocko7ffc0ed2012-10-08 16:33:13 -0700483static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
484{
485 return (memcg == root_mem_cgroup);
486}
487
Glauber Costae1aab162011-12-11 21:47:03 +0000488/* Writing them here to avoid exposing memcg's inner layout */
Michal Hocko4bd2c1e2012-10-08 16:33:10 -0700489#if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
Glauber Costae1aab162011-12-11 21:47:03 +0000490
Glauber Costae1aab162011-12-11 21:47:03 +0000491void sock_update_memcg(struct sock *sk)
492{
Glauber Costa376be5f2012-01-20 04:57:14 +0000493 if (mem_cgroup_sockets_enabled) {
Glauber Costae1aab162011-12-11 21:47:03 +0000494 struct mem_cgroup *memcg;
Glauber Costa3f134612012-05-29 15:07:11 -0700495 struct cg_proto *cg_proto;
Glauber Costae1aab162011-12-11 21:47:03 +0000496
497 BUG_ON(!sk->sk_prot->proto_cgroup);
498
Glauber Costaf3f511e2012-01-05 20:16:39 +0000499 /* Socket cloning can throw us here with sk_cgrp already
500 * filled. It won't however, necessarily happen from
501 * process context. So the test for root memcg given
502 * the current task's memcg won't help us in this case.
503 *
504 * Respecting the original socket's memcg is a better
505 * decision in this case.
506 */
507 if (sk->sk_cgrp) {
508 BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
509 mem_cgroup_get(sk->sk_cgrp->memcg);
510 return;
511 }
512
Glauber Costae1aab162011-12-11 21:47:03 +0000513 rcu_read_lock();
514 memcg = mem_cgroup_from_task(current);
Glauber Costa3f134612012-05-29 15:07:11 -0700515 cg_proto = sk->sk_prot->proto_cgroup(memcg);
516 if (!mem_cgroup_is_root(memcg) && memcg_proto_active(cg_proto)) {
Glauber Costae1aab162011-12-11 21:47:03 +0000517 mem_cgroup_get(memcg);
Glauber Costa3f134612012-05-29 15:07:11 -0700518 sk->sk_cgrp = cg_proto;
Glauber Costae1aab162011-12-11 21:47:03 +0000519 }
520 rcu_read_unlock();
521 }
522}
523EXPORT_SYMBOL(sock_update_memcg);
524
525void sock_release_memcg(struct sock *sk)
526{
Glauber Costa376be5f2012-01-20 04:57:14 +0000527 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
Glauber Costae1aab162011-12-11 21:47:03 +0000528 struct mem_cgroup *memcg;
529 WARN_ON(!sk->sk_cgrp->memcg);
530 memcg = sk->sk_cgrp->memcg;
531 mem_cgroup_put(memcg);
532 }
533}
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000534
535struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
536{
537 if (!memcg || mem_cgroup_is_root(memcg))
538 return NULL;
539
540 return &memcg->tcp_mem.cg_proto;
541}
542EXPORT_SYMBOL(tcp_proto_cgroup);
Glauber Costae1aab162011-12-11 21:47:03 +0000543
Glauber Costa3f134612012-05-29 15:07:11 -0700544static void disarm_sock_keys(struct mem_cgroup *memcg)
545{
546 if (!memcg_proto_activated(&memcg->tcp_mem.cg_proto))
547 return;
548 static_key_slow_dec(&memcg_socket_limit_enabled);
549}
550#else
551static void disarm_sock_keys(struct mem_cgroup *memcg)
552{
553}
554#endif
555
Glauber Costaa8964b92012-12-18 14:22:09 -0800556#ifdef CONFIG_MEMCG_KMEM
Glauber Costa55007d82012-12-18 14:22:38 -0800557/*
558 * This will be the memcg's index in each cache's ->memcg_params->memcg_caches.
559 * There are two main reasons for not using the css_id for this:
560 * 1) this works better in sparse environments, where we have a lot of memcgs,
561 * but only a few kmem-limited. Or also, if we have, for instance, 200
562 * memcgs, and none but the 200th is kmem-limited, we'd have to have a
563 * 200 entry array for that.
564 *
565 * 2) In order not to violate the cgroup API, we would like to do all memory
566 * allocation in ->create(). At that point, we haven't yet allocated the
567 * css_id. Having a separate index prevents us from messing with the cgroup
568 * core for this
569 *
570 * The current size of the caches array is stored in
571 * memcg_limited_groups_array_size. It will double each time we have to
572 * increase it.
573 */
574static DEFINE_IDA(kmem_limited_groups);
Glauber Costa749c5412012-12-18 14:23:01 -0800575int memcg_limited_groups_array_size;
576
Glauber Costa55007d82012-12-18 14:22:38 -0800577/*
578 * MIN_SIZE is different than 1, because we would like to avoid going through
579 * the alloc/free process all the time. In a small machine, 4 kmem-limited
580 * cgroups is a reasonable guess. In the future, it could be a parameter or
581 * tunable, but that is strictly not necessary.
582 *
583 * MAX_SIZE should be as large as the number of css_ids. Ideally, we could get
584 * this constant directly from cgroup, but it is understandable that this is
585 * better kept as an internal representation in cgroup.c. In any case, the
586 * css_id space is not getting any smaller, and we don't have to necessarily
587 * increase ours as well if it increases.
588 */
589#define MEMCG_CACHES_MIN_SIZE 4
590#define MEMCG_CACHES_MAX_SIZE 65535
591
Glauber Costad7f25f82012-12-18 14:22:40 -0800592/*
593 * A lot of the calls to the cache allocation functions are expected to be
594 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
595 * conditional to this static branch, we'll have to allow modules that does
596 * kmem_cache_alloc and the such to see this symbol as well
597 */
Glauber Costaa8964b92012-12-18 14:22:09 -0800598struct static_key memcg_kmem_enabled_key;
Glauber Costad7f25f82012-12-18 14:22:40 -0800599EXPORT_SYMBOL(memcg_kmem_enabled_key);
Glauber Costaa8964b92012-12-18 14:22:09 -0800600
601static void disarm_kmem_keys(struct mem_cgroup *memcg)
602{
Glauber Costa55007d82012-12-18 14:22:38 -0800603 if (memcg_kmem_is_active(memcg)) {
Glauber Costaa8964b92012-12-18 14:22:09 -0800604 static_key_slow_dec(&memcg_kmem_enabled_key);
Glauber Costa55007d82012-12-18 14:22:38 -0800605 ida_simple_remove(&kmem_limited_groups, memcg->kmemcg_id);
606 }
Glauber Costabea207c2012-12-18 14:22:11 -0800607 /*
608 * This check can't live in kmem destruction function,
609 * since the charges will outlive the cgroup
610 */
611 WARN_ON(res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0);
Glauber Costaa8964b92012-12-18 14:22:09 -0800612}
613#else
614static void disarm_kmem_keys(struct mem_cgroup *memcg)
615{
616}
617#endif /* CONFIG_MEMCG_KMEM */
618
619static void disarm_static_keys(struct mem_cgroup *memcg)
620{
621 disarm_sock_keys(memcg);
622 disarm_kmem_keys(memcg);
623}
624
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700625static void drain_all_stock_async(struct mem_cgroup *memcg);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800626
Balbir Singhf64c3f52009-09-23 15:56:37 -0700627static struct mem_cgroup_per_zone *
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700628mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700629{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700630 return &memcg->info.nodeinfo[nid]->zoneinfo[zid];
Balbir Singhf64c3f52009-09-23 15:56:37 -0700631}
632
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700633struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
Wu Fengguangd3242362009-12-16 12:19:59 +0100634{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700635 return &memcg->css;
Wu Fengguangd3242362009-12-16 12:19:59 +0100636}
637
Balbir Singhf64c3f52009-09-23 15:56:37 -0700638static struct mem_cgroup_per_zone *
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700639page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700640{
Johannes Weiner97a6c372011-03-23 16:42:27 -0700641 int nid = page_to_nid(page);
642 int zid = page_zonenum(page);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700643
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700644 return mem_cgroup_zoneinfo(memcg, nid, zid);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700645}
646
647static struct mem_cgroup_tree_per_zone *
648soft_limit_tree_node_zone(int nid, int zid)
649{
650 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
651}
652
653static struct mem_cgroup_tree_per_zone *
654soft_limit_tree_from_page(struct page *page)
655{
656 int nid = page_to_nid(page);
657 int zid = page_zonenum(page);
658
659 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
660}
661
662static void
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700663__mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
Balbir Singhf64c3f52009-09-23 15:56:37 -0700664 struct mem_cgroup_per_zone *mz,
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700665 struct mem_cgroup_tree_per_zone *mctz,
666 unsigned long long new_usage_in_excess)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700667{
668 struct rb_node **p = &mctz->rb_root.rb_node;
669 struct rb_node *parent = NULL;
670 struct mem_cgroup_per_zone *mz_node;
671
672 if (mz->on_tree)
673 return;
674
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700675 mz->usage_in_excess = new_usage_in_excess;
676 if (!mz->usage_in_excess)
677 return;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700678 while (*p) {
679 parent = *p;
680 mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
681 tree_node);
682 if (mz->usage_in_excess < mz_node->usage_in_excess)
683 p = &(*p)->rb_left;
684 /*
685 * We can't avoid mem cgroups that are over their soft
686 * limit by the same amount
687 */
688 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
689 p = &(*p)->rb_right;
690 }
691 rb_link_node(&mz->tree_node, parent, p);
692 rb_insert_color(&mz->tree_node, &mctz->rb_root);
693 mz->on_tree = true;
Balbir Singh4e416952009-09-23 15:56:39 -0700694}
695
696static void
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700697__mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
Balbir Singh4e416952009-09-23 15:56:39 -0700698 struct mem_cgroup_per_zone *mz,
699 struct mem_cgroup_tree_per_zone *mctz)
700{
701 if (!mz->on_tree)
702 return;
703 rb_erase(&mz->tree_node, &mctz->rb_root);
704 mz->on_tree = false;
705}
706
707static void
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700708mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
Balbir Singhf64c3f52009-09-23 15:56:37 -0700709 struct mem_cgroup_per_zone *mz,
710 struct mem_cgroup_tree_per_zone *mctz)
711{
712 spin_lock(&mctz->lock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700713 __mem_cgroup_remove_exceeded(memcg, mz, mctz);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700714 spin_unlock(&mctz->lock);
715}
716
Balbir Singhf64c3f52009-09-23 15:56:37 -0700717
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700718static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700719{
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700720 unsigned long long excess;
Balbir Singhf64c3f52009-09-23 15:56:37 -0700721 struct mem_cgroup_per_zone *mz;
722 struct mem_cgroup_tree_per_zone *mctz;
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700723 int nid = page_to_nid(page);
724 int zid = page_zonenum(page);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700725 mctz = soft_limit_tree_from_page(page);
726
727 /*
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700728 * Necessary to update all ancestors when hierarchy is used.
729 * because their event counter is not touched.
Balbir Singhf64c3f52009-09-23 15:56:37 -0700730 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700731 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
732 mz = mem_cgroup_zoneinfo(memcg, nid, zid);
733 excess = res_counter_soft_limit_excess(&memcg->res);
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700734 /*
735 * We have to update the tree if mz is on RB-tree or
736 * mem is over its softlimit.
737 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700738 if (excess || mz->on_tree) {
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700739 spin_lock(&mctz->lock);
740 /* if on-tree, remove it */
741 if (mz->on_tree)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700742 __mem_cgroup_remove_exceeded(memcg, mz, mctz);
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700743 /*
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -0700744 * Insert again. mz->usage_in_excess will be updated.
745 * If excess is 0, no tree ops.
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700746 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700747 __mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -0700748 spin_unlock(&mctz->lock);
749 }
Balbir Singhf64c3f52009-09-23 15:56:37 -0700750 }
751}
752
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700753static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700754{
755 int node, zone;
756 struct mem_cgroup_per_zone *mz;
757 struct mem_cgroup_tree_per_zone *mctz;
758
Bob Liu3ed28fa2012-01-12 17:19:04 -0800759 for_each_node(node) {
Balbir Singhf64c3f52009-09-23 15:56:37 -0700760 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700761 mz = mem_cgroup_zoneinfo(memcg, node, zone);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700762 mctz = soft_limit_tree_node_zone(node, zone);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700763 mem_cgroup_remove_exceeded(memcg, mz, mctz);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700764 }
765 }
766}
767
Balbir Singh4e416952009-09-23 15:56:39 -0700768static struct mem_cgroup_per_zone *
769__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
770{
771 struct rb_node *rightmost = NULL;
KAMEZAWA Hiroyuki26251ea2009-10-01 15:44:08 -0700772 struct mem_cgroup_per_zone *mz;
Balbir Singh4e416952009-09-23 15:56:39 -0700773
774retry:
KAMEZAWA Hiroyuki26251ea2009-10-01 15:44:08 -0700775 mz = NULL;
Balbir Singh4e416952009-09-23 15:56:39 -0700776 rightmost = rb_last(&mctz->rb_root);
777 if (!rightmost)
778 goto done; /* Nothing to reclaim from */
779
780 mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
781 /*
782 * Remove the node now but someone else can add it back,
783 * we will to add it back at the end of reclaim to its correct
784 * position in the tree.
785 */
Hugh Dickinsd79154b2012-03-21 16:34:18 -0700786 __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
787 if (!res_counter_soft_limit_excess(&mz->memcg->res) ||
788 !css_tryget(&mz->memcg->css))
Balbir Singh4e416952009-09-23 15:56:39 -0700789 goto retry;
790done:
791 return mz;
792}
793
794static struct mem_cgroup_per_zone *
795mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
796{
797 struct mem_cgroup_per_zone *mz;
798
799 spin_lock(&mctz->lock);
800 mz = __mem_cgroup_largest_soft_limit_node(mctz);
801 spin_unlock(&mctz->lock);
802 return mz;
803}
804
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700805/*
806 * Implementation Note: reading percpu statistics for memcg.
807 *
808 * Both of vmstat[] and percpu_counter has threshold and do periodic
809 * synchronization to implement "quick" read. There are trade-off between
810 * reading cost and precision of value. Then, we may have a chance to implement
811 * a periodic synchronizion of counter in memcg's counter.
812 *
813 * But this _read() function is used for user interface now. The user accounts
814 * memory usage by memory cgroup and he _always_ requires exact value because
815 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
816 * have to visit all online cpus and make sum. So, for now, unnecessary
817 * synchronization is not implemented. (just implemented for cpu hotplug)
818 *
819 * If there are kernel internal actions which can make use of some not-exact
820 * value, and reading all cpu value can be performance bottleneck in some
821 * common workload, threashold and synchonization as vmstat[] should be
822 * implemented.
823 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700824static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700825 enum mem_cgroup_stat_index idx)
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800826{
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700827 long val = 0;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800828 int cpu;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800829
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700830 get_online_cpus();
831 for_each_online_cpu(cpu)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700832 val += per_cpu(memcg->stat->count[idx], cpu);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700833#ifdef CONFIG_HOTPLUG_CPU
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700834 spin_lock(&memcg->pcp_counter_lock);
835 val += memcg->nocpu_base.count[idx];
836 spin_unlock(&memcg->pcp_counter_lock);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700837#endif
838 put_online_cpus();
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800839 return val;
840}
841
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700842static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700843 bool charge)
844{
845 int val = (charge) ? 1 : -1;
Kamezawa Hiroyukibff6bb82012-07-31 16:41:38 -0700846 this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700847}
848
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700849static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
Johannes Weinere9f89742011-03-23 16:42:37 -0700850 enum mem_cgroup_events_index idx)
851{
852 unsigned long val = 0;
853 int cpu;
854
855 for_each_online_cpu(cpu)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700856 val += per_cpu(memcg->stat->events[idx], cpu);
Johannes Weinere9f89742011-03-23 16:42:37 -0700857#ifdef CONFIG_HOTPLUG_CPU
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700858 spin_lock(&memcg->pcp_counter_lock);
859 val += memcg->nocpu_base.events[idx];
860 spin_unlock(&memcg->pcp_counter_lock);
Johannes Weinere9f89742011-03-23 16:42:37 -0700861#endif
862 return val;
863}
864
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700865static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -0700866 bool anon, int nr_pages)
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800867{
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800868 preempt_disable();
869
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -0700870 /*
871 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
872 * counted as CACHE even if it's on ANON LRU.
873 */
874 if (anon)
875 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700876 nr_pages);
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800877 else
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -0700878 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700879 nr_pages);
Balaji Rao55e462b2008-05-01 04:35:12 -0700880
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800881 /* pagein of a big page is an event. So, ignore page size */
882 if (nr_pages > 0)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700883 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
KAMEZAWA Hiroyuki3751d602011-02-01 15:52:45 -0800884 else {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700885 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
KAMEZAWA Hiroyuki3751d602011-02-01 15:52:45 -0800886 nr_pages = -nr_pages; /* for event */
887 }
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800888
Johannes Weiner13114712012-05-29 15:07:07 -0700889 __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800890
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800891 preempt_enable();
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800892}
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800893
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700894unsigned long
Hugh Dickins4d7dcca2012-05-29 15:07:08 -0700895mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
Konstantin Khlebnikov074291f2012-05-29 15:07:00 -0700896{
897 struct mem_cgroup_per_zone *mz;
898
899 mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
900 return mz->lru_size[lru];
901}
902
903static unsigned long
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700904mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700905 unsigned int lru_mask)
Ying Han889976d2011-05-26 16:25:33 -0700906{
907 struct mem_cgroup_per_zone *mz;
Hugh Dickinsf156ab92012-03-21 16:34:19 -0700908 enum lru_list lru;
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700909 unsigned long ret = 0;
910
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700911 mz = mem_cgroup_zoneinfo(memcg, nid, zid);
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700912
Hugh Dickinsf156ab92012-03-21 16:34:19 -0700913 for_each_lru(lru) {
914 if (BIT(lru) & lru_mask)
915 ret += mz->lru_size[lru];
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700916 }
917 return ret;
918}
919
920static unsigned long
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700921mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700922 int nid, unsigned int lru_mask)
923{
Ying Han889976d2011-05-26 16:25:33 -0700924 u64 total = 0;
925 int zid;
926
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700927 for (zid = 0; zid < MAX_NR_ZONES; zid++)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700928 total += mem_cgroup_zone_nr_lru_pages(memcg,
929 nid, zid, lru_mask);
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700930
Ying Han889976d2011-05-26 16:25:33 -0700931 return total;
932}
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700933
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700934static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700935 unsigned int lru_mask)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800936{
Ying Han889976d2011-05-26 16:25:33 -0700937 int nid;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800938 u64 total = 0;
939
Lai Jiangshan31aaea42012-12-12 13:51:27 -0800940 for_each_node_state(nid, N_MEMORY)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700941 total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800942 return total;
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800943}
944
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800945static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
946 enum mem_cgroup_events_target target)
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800947{
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700948 unsigned long val, next;
949
Johannes Weiner13114712012-05-29 15:07:07 -0700950 val = __this_cpu_read(memcg->stat->nr_page_events);
Steven Rostedt47994012011-11-02 13:38:33 -0700951 next = __this_cpu_read(memcg->stat->targets[target]);
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700952 /* from time_after() in jiffies.h */
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800953 if ((long)next - (long)val < 0) {
954 switch (target) {
955 case MEM_CGROUP_TARGET_THRESH:
956 next = val + THRESHOLDS_EVENTS_TARGET;
957 break;
958 case MEM_CGROUP_TARGET_SOFTLIMIT:
959 next = val + SOFTLIMIT_EVENTS_TARGET;
960 break;
961 case MEM_CGROUP_TARGET_NUMAINFO:
962 next = val + NUMAINFO_EVENTS_TARGET;
963 break;
964 default:
965 break;
966 }
967 __this_cpu_write(memcg->stat->targets[target], next);
968 return true;
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700969 }
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800970 return false;
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800971}
972
973/*
974 * Check events in order.
975 *
976 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700977static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800978{
Steven Rostedt47994012011-11-02 13:38:33 -0700979 preempt_disable();
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800980 /* threshold event is triggered in finer grain than soft limit */
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800981 if (unlikely(mem_cgroup_event_ratelimit(memcg,
982 MEM_CGROUP_TARGET_THRESH))) {
Andrew Morton82b3f2a2012-02-03 15:37:14 -0800983 bool do_softlimit;
984 bool do_numainfo __maybe_unused;
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800985
986 do_softlimit = mem_cgroup_event_ratelimit(memcg,
987 MEM_CGROUP_TARGET_SOFTLIMIT);
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -0700988#if MAX_NUMNODES > 1
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800989 do_numainfo = mem_cgroup_event_ratelimit(memcg,
990 MEM_CGROUP_TARGET_NUMAINFO);
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -0700991#endif
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800992 preempt_enable();
993
994 mem_cgroup_threshold(memcg);
995 if (unlikely(do_softlimit))
996 mem_cgroup_update_tree(memcg, page);
997#if MAX_NUMNODES > 1
998 if (unlikely(do_numainfo))
999 atomic_inc(&memcg->numainfo_events);
1000#endif
1001 } else
1002 preempt_enable();
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08001003}
1004
Glauber Costad1a4c0b2011-12-11 21:47:04 +00001005struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001006{
Wanpeng Lib2145142012-07-31 16:46:01 -07001007 return mem_cgroup_from_css(
1008 cgroup_subsys_state(cont, mem_cgroup_subsys_id));
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001009}
1010
Balbir Singhcf475ad2008-04-29 01:00:16 -07001011struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001012{
Balbir Singh31a78f22008-09-28 23:09:31 +01001013 /*
1014 * mm_update_next_owner() may clear mm->owner to NULL
1015 * if it races with swapoff, page migration, etc.
1016 * So this can be called with p == NULL.
1017 */
1018 if (unlikely(!p))
1019 return NULL;
1020
Wanpeng Lib2145142012-07-31 16:46:01 -07001021 return mem_cgroup_from_css(task_subsys_state(p, mem_cgroup_subsys_id));
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001022}
1023
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07001024struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001025{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001026 struct mem_cgroup *memcg = NULL;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001027
1028 if (!mm)
1029 return NULL;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001030 /*
1031 * Because we have no locks, mm->owner's may be being moved to other
1032 * cgroup. We use css_tryget() here even if this looks
1033 * pessimistic (rather than adding locks here).
1034 */
1035 rcu_read_lock();
1036 do {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001037 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1038 if (unlikely(!memcg))
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001039 break;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001040 } while (!css_tryget(&memcg->css));
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001041 rcu_read_unlock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001042 return memcg;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001043}
1044
Johannes Weiner56600482012-01-12 17:17:59 -08001045/**
1046 * mem_cgroup_iter - iterate over memory cgroup hierarchy
1047 * @root: hierarchy root
1048 * @prev: previously returned memcg, NULL on first invocation
1049 * @reclaim: cookie for shared reclaim walks, NULL for full walks
1050 *
1051 * Returns references to children of the hierarchy below @root, or
1052 * @root itself, or %NULL after a full round-trip.
1053 *
1054 * Caller must pass the return value in @prev on subsequent
1055 * invocations for reference counting, or use mem_cgroup_iter_break()
1056 * to cancel a hierarchy walk before the round-trip is complete.
1057 *
1058 * Reclaimers can specify a zone and a priority level in @reclaim to
1059 * divide up the memcgs in the hierarchy among all concurrent
1060 * reclaimers operating on the same zone and priority.
1061 */
1062struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
1063 struct mem_cgroup *prev,
1064 struct mem_cgroup_reclaim_cookie *reclaim)
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07001065{
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001066 struct mem_cgroup *memcg = NULL;
1067 int id = 0;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001068
Johannes Weiner56600482012-01-12 17:17:59 -08001069 if (mem_cgroup_disabled())
1070 return NULL;
1071
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001072 if (!root)
1073 root = root_mem_cgroup;
1074
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001075 if (prev && !reclaim)
1076 id = css_id(&prev->css);
1077
1078 if (prev && prev != root)
1079 css_put(&prev->css);
1080
1081 if (!root->use_hierarchy && root != root_mem_cgroup) {
1082 if (prev)
1083 return NULL;
1084 return root;
1085 }
1086
1087 while (!memcg) {
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001088 struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001089 struct cgroup_subsys_state *css;
1090
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001091 if (reclaim) {
1092 int nid = zone_to_nid(reclaim->zone);
1093 int zid = zone_idx(reclaim->zone);
1094 struct mem_cgroup_per_zone *mz;
1095
1096 mz = mem_cgroup_zoneinfo(root, nid, zid);
1097 iter = &mz->reclaim_iter[reclaim->priority];
1098 if (prev && reclaim->generation != iter->generation)
1099 return NULL;
1100 id = iter->position;
1101 }
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001102
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001103 rcu_read_lock();
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001104 css = css_get_next(&mem_cgroup_subsys, id + 1, &root->css, &id);
1105 if (css) {
1106 if (css == &root->css || css_tryget(css))
Wanpeng Lib2145142012-07-31 16:46:01 -07001107 memcg = mem_cgroup_from_css(css);
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001108 } else
1109 id = 0;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001110 rcu_read_unlock();
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001111
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001112 if (reclaim) {
1113 iter->position = id;
1114 if (!css)
1115 iter->generation++;
1116 else if (!prev && memcg)
1117 reclaim->generation = iter->generation;
1118 }
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001119
1120 if (prev && !css)
1121 return NULL;
1122 }
1123 return memcg;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001124}
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001125
Johannes Weiner56600482012-01-12 17:17:59 -08001126/**
1127 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1128 * @root: hierarchy root
1129 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1130 */
1131void mem_cgroup_iter_break(struct mem_cgroup *root,
1132 struct mem_cgroup *prev)
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001133{
1134 if (!root)
1135 root = root_mem_cgroup;
1136 if (prev && prev != root)
1137 css_put(&prev->css);
1138}
1139
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001140/*
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001141 * Iteration constructs for visiting all cgroups (under a tree). If
1142 * loops are exited prematurely (break), mem_cgroup_iter_break() must
1143 * be used for reference counting.
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001144 */
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001145#define for_each_mem_cgroup_tree(iter, root) \
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001146 for (iter = mem_cgroup_iter(root, NULL, NULL); \
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001147 iter != NULL; \
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001148 iter = mem_cgroup_iter(root, iter, NULL))
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001149
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001150#define for_each_mem_cgroup(iter) \
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001151 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001152 iter != NULL; \
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001153 iter = mem_cgroup_iter(NULL, iter, NULL))
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001154
David Rientjes68ae5642012-12-12 13:51:57 -08001155void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
Ying Han456f9982011-05-26 16:25:38 -07001156{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001157 struct mem_cgroup *memcg;
Ying Han456f9982011-05-26 16:25:38 -07001158
Ying Han456f9982011-05-26 16:25:38 -07001159 rcu_read_lock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001160 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1161 if (unlikely(!memcg))
Ying Han456f9982011-05-26 16:25:38 -07001162 goto out;
1163
1164 switch (idx) {
Ying Han456f9982011-05-26 16:25:38 -07001165 case PGFAULT:
Johannes Weiner0e574a92012-01-12 17:18:35 -08001166 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
1167 break;
1168 case PGMAJFAULT:
1169 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
Ying Han456f9982011-05-26 16:25:38 -07001170 break;
1171 default:
1172 BUG();
1173 }
1174out:
1175 rcu_read_unlock();
1176}
David Rientjes68ae5642012-12-12 13:51:57 -08001177EXPORT_SYMBOL(__mem_cgroup_count_vm_event);
Ying Han456f9982011-05-26 16:25:38 -07001178
Johannes Weiner925b7672012-01-12 17:18:15 -08001179/**
1180 * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
1181 * @zone: zone of the wanted lruvec
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001182 * @memcg: memcg of the wanted lruvec
Johannes Weiner925b7672012-01-12 17:18:15 -08001183 *
1184 * Returns the lru list vector holding pages for the given @zone and
1185 * @mem. This can be the global zone lruvec, if the memory controller
1186 * is disabled.
1187 */
1188struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
1189 struct mem_cgroup *memcg)
1190{
1191 struct mem_cgroup_per_zone *mz;
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001192 struct lruvec *lruvec;
Johannes Weiner925b7672012-01-12 17:18:15 -08001193
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001194 if (mem_cgroup_disabled()) {
1195 lruvec = &zone->lruvec;
1196 goto out;
1197 }
Johannes Weiner925b7672012-01-12 17:18:15 -08001198
1199 mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001200 lruvec = &mz->lruvec;
1201out:
1202 /*
1203 * Since a node can be onlined after the mem_cgroup was created,
1204 * we have to be prepared to initialize lruvec->zone here;
1205 * and if offlined then reonlined, we need to reinitialize it.
1206 */
1207 if (unlikely(lruvec->zone != zone))
1208 lruvec->zone = zone;
1209 return lruvec;
Johannes Weiner925b7672012-01-12 17:18:15 -08001210}
1211
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001212/*
1213 * Following LRU functions are allowed to be used without PCG_LOCK.
1214 * Operations are called by routine of global LRU independently from memcg.
1215 * What we have to take care of here is validness of pc->mem_cgroup.
1216 *
1217 * Changes to pc->mem_cgroup happens when
1218 * 1. charge
1219 * 2. moving account
1220 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
1221 * It is added to LRU before charge.
1222 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
1223 * When moving account, the page is not on LRU. It's isolated.
1224 */
1225
Johannes Weiner925b7672012-01-12 17:18:15 -08001226/**
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001227 * mem_cgroup_page_lruvec - return lruvec for adding an lru page
Johannes Weiner925b7672012-01-12 17:18:15 -08001228 * @page: the page
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001229 * @zone: zone of the page
Minchan Kim3f58a822011-03-22 16:32:53 -07001230 */
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001231struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
Minchan Kim3f58a822011-03-22 16:32:53 -07001232{
1233 struct mem_cgroup_per_zone *mz;
Johannes Weiner925b7672012-01-12 17:18:15 -08001234 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001235 struct page_cgroup *pc;
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001236 struct lruvec *lruvec;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001237
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001238 if (mem_cgroup_disabled()) {
1239 lruvec = &zone->lruvec;
1240 goto out;
1241 }
Christoph Lameterb69408e2008-10-18 20:26:14 -07001242
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001243 pc = lookup_page_cgroup(page);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08001244 memcg = pc->mem_cgroup;
Hugh Dickins75121022012-03-05 14:59:18 -08001245
1246 /*
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001247 * Surreptitiously switch any uncharged offlist page to root:
Hugh Dickins75121022012-03-05 14:59:18 -08001248 * an uncharged page off lru does nothing to secure
1249 * its former mem_cgroup from sudden removal.
1250 *
1251 * Our caller holds lru_lock, and PageCgroupUsed is updated
1252 * under page_cgroup lock: between them, they make all uses
1253 * of pc->mem_cgroup safe.
1254 */
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001255 if (!PageLRU(page) && !PageCgroupUsed(pc) && memcg != root_mem_cgroup)
Hugh Dickins75121022012-03-05 14:59:18 -08001256 pc->mem_cgroup = memcg = root_mem_cgroup;
1257
Johannes Weiner925b7672012-01-12 17:18:15 -08001258 mz = page_cgroup_zoneinfo(memcg, page);
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001259 lruvec = &mz->lruvec;
1260out:
1261 /*
1262 * Since a node can be onlined after the mem_cgroup was created,
1263 * we have to be prepared to initialize lruvec->zone here;
1264 * and if offlined then reonlined, we need to reinitialize it.
1265 */
1266 if (unlikely(lruvec->zone != zone))
1267 lruvec->zone = zone;
1268 return lruvec;
Johannes Weiner925b7672012-01-12 17:18:15 -08001269}
1270
1271/**
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001272 * mem_cgroup_update_lru_size - account for adding or removing an lru page
1273 * @lruvec: mem_cgroup per zone lru vector
1274 * @lru: index of lru list the page is sitting on
1275 * @nr_pages: positive when adding or negative when removing
Johannes Weiner925b7672012-01-12 17:18:15 -08001276 *
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001277 * This function must be called when a page is added to or removed from an
1278 * lru list.
Johannes Weiner925b7672012-01-12 17:18:15 -08001279 */
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001280void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1281 int nr_pages)
Johannes Weiner925b7672012-01-12 17:18:15 -08001282{
1283 struct mem_cgroup_per_zone *mz;
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001284 unsigned long *lru_size;
Johannes Weiner925b7672012-01-12 17:18:15 -08001285
1286 if (mem_cgroup_disabled())
1287 return;
1288
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001289 mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
1290 lru_size = mz->lru_size + lru;
1291 *lru_size += nr_pages;
1292 VM_BUG_ON((long)(*lru_size) < 0);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001293}
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08001294
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001295/*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001296 * Checks whether given mem is same or in the root_mem_cgroup's
Michal Hocko3e920412011-07-26 16:08:29 -07001297 * hierarchy subtree
1298 */
Johannes Weinerc3ac9a82012-05-29 15:06:25 -07001299bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
1300 struct mem_cgroup *memcg)
Michal Hocko3e920412011-07-26 16:08:29 -07001301{
Johannes Weiner91c637342012-05-29 15:06:24 -07001302 if (root_memcg == memcg)
1303 return true;
Hugh Dickins3a981f42012-06-20 12:52:58 -07001304 if (!root_memcg->use_hierarchy || !memcg)
Johannes Weiner91c637342012-05-29 15:06:24 -07001305 return false;
Johannes Weinerc3ac9a82012-05-29 15:06:25 -07001306 return css_is_ancestor(&memcg->css, &root_memcg->css);
1307}
1308
1309static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
1310 struct mem_cgroup *memcg)
1311{
1312 bool ret;
1313
Johannes Weiner91c637342012-05-29 15:06:24 -07001314 rcu_read_lock();
Johannes Weinerc3ac9a82012-05-29 15:06:25 -07001315 ret = __mem_cgroup_same_or_subtree(root_memcg, memcg);
Johannes Weiner91c637342012-05-29 15:06:24 -07001316 rcu_read_unlock();
1317 return ret;
Michal Hocko3e920412011-07-26 16:08:29 -07001318}
1319
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001320int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg)
David Rientjes4c4a2212008-02-07 00:14:06 -08001321{
1322 int ret;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001323 struct mem_cgroup *curr = NULL;
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -07001324 struct task_struct *p;
David Rientjes4c4a2212008-02-07 00:14:06 -08001325
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -07001326 p = find_lock_task_mm(task);
David Rientjesde077d22012-01-12 17:18:52 -08001327 if (p) {
1328 curr = try_get_mem_cgroup_from_mm(p->mm);
1329 task_unlock(p);
1330 } else {
1331 /*
1332 * All threads may have already detached their mm's, but the oom
1333 * killer still needs to detect if they have already been oom
1334 * killed to prevent needlessly killing additional tasks.
1335 */
1336 task_lock(task);
1337 curr = mem_cgroup_from_task(task);
1338 if (curr)
1339 css_get(&curr->css);
1340 task_unlock(task);
1341 }
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001342 if (!curr)
1343 return 0;
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -08001344 /*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001345 * We should check use_hierarchy of "memcg" not "curr". Because checking
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -08001346 * use_hierarchy of "curr" here make this function true if hierarchy is
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001347 * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
1348 * hierarchy(even if use_hierarchy is disabled in "memcg").
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -08001349 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001350 ret = mem_cgroup_same_or_subtree(memcg, curr);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001351 css_put(&curr->css);
David Rientjes4c4a2212008-02-07 00:14:06 -08001352 return ret;
1353}
1354
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07001355int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001356{
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08001357 unsigned long inactive_ratio;
Johannes Weiner9b272972011-11-02 13:38:23 -07001358 unsigned long inactive;
1359 unsigned long active;
1360 unsigned long gb;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001361
Hugh Dickins4d7dcca2012-05-29 15:07:08 -07001362 inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
1363 active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001364
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08001365 gb = (inactive + active) >> (30 - PAGE_SHIFT);
1366 if (gb)
1367 inactive_ratio = int_sqrt(10 * gb);
1368 else
1369 inactive_ratio = 1;
1370
Johannes Weiner9b272972011-11-02 13:38:23 -07001371 return inactive * inactive_ratio < active;
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08001372}
1373
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07001374int mem_cgroup_inactive_file_is_low(struct lruvec *lruvec)
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08001375{
1376 unsigned long active;
1377 unsigned long inactive;
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08001378
Hugh Dickins4d7dcca2012-05-29 15:07:08 -07001379 inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_FILE);
1380 active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_FILE);
Rik van Riel56e49d22009-06-16 15:32:28 -07001381
1382 return (active > inactive);
1383}
1384
Balbir Singh6d61ef42009-01-07 18:08:06 -08001385#define mem_cgroup_from_res_counter(counter, member) \
1386 container_of(counter, struct mem_cgroup, member)
1387
Johannes Weiner19942822011-02-01 15:52:43 -08001388/**
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001389 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
Wanpeng Lidad75572012-06-20 12:53:01 -07001390 * @memcg: the memory cgroup
Johannes Weiner19942822011-02-01 15:52:43 -08001391 *
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001392 * Returns the maximum amount of memory @mem can be charged with, in
Johannes Weiner7ec99d62011-03-23 16:42:36 -07001393 * pages.
Johannes Weiner19942822011-02-01 15:52:43 -08001394 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001395static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
Johannes Weiner19942822011-02-01 15:52:43 -08001396{
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001397 unsigned long long margin;
1398
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001399 margin = res_counter_margin(&memcg->res);
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001400 if (do_swap_account)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001401 margin = min(margin, res_counter_margin(&memcg->memsw));
Johannes Weiner7ec99d62011-03-23 16:42:36 -07001402 return margin >> PAGE_SHIFT;
Johannes Weiner19942822011-02-01 15:52:43 -08001403}
1404
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07001405int mem_cgroup_swappiness(struct mem_cgroup *memcg)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001406{
1407 struct cgroup *cgrp = memcg->css.cgroup;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001408
1409 /* root ? */
1410 if (cgrp->parent == NULL)
1411 return vm_swappiness;
1412
Johannes Weinerbf1ff262011-03-23 16:42:32 -07001413 return memcg->swappiness;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001414}
1415
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001416/*
1417 * memcg->moving_account is used for checking possibility that some thread is
1418 * calling move_account(). When a thread on CPU-A starts moving pages under
1419 * a memcg, other threads should check memcg->moving_account under
1420 * rcu_read_lock(), like this:
1421 *
1422 * CPU-A CPU-B
1423 * rcu_read_lock()
1424 * memcg->moving_account+1 if (memcg->mocing_account)
1425 * take heavy locks.
1426 * synchronize_rcu() update something.
1427 * rcu_read_unlock()
1428 * start move here.
1429 */
KAMEZAWA Hiroyuki4331f7d2012-03-21 16:34:26 -07001430
1431/* for quick checking without looking up memcg */
1432atomic_t memcg_moving __read_mostly;
1433
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001434static void mem_cgroup_start_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001435{
KAMEZAWA Hiroyuki4331f7d2012-03-21 16:34:26 -07001436 atomic_inc(&memcg_moving);
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001437 atomic_inc(&memcg->moving_account);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001438 synchronize_rcu();
1439}
1440
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001441static void mem_cgroup_end_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001442{
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001443 /*
1444 * Now, mem_cgroup_clear_mc() may call this function with NULL.
1445 * We check NULL in callee rather than caller.
1446 */
KAMEZAWA Hiroyuki4331f7d2012-03-21 16:34:26 -07001447 if (memcg) {
1448 atomic_dec(&memcg_moving);
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001449 atomic_dec(&memcg->moving_account);
KAMEZAWA Hiroyuki4331f7d2012-03-21 16:34:26 -07001450 }
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001451}
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001452
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001453/*
1454 * 2 routines for checking "mem" is under move_account() or not.
1455 *
Andrew Morton13fd1dd92012-03-21 16:34:26 -07001456 * mem_cgroup_stolen() - checking whether a cgroup is mc.from or not. This
1457 * is used for avoiding races in accounting. If true,
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001458 * pc->mem_cgroup may be overwritten.
1459 *
1460 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
1461 * under hierarchy of moving cgroups. This is for
1462 * waiting at hith-memory prressure caused by "move".
1463 */
1464
Andrew Morton13fd1dd92012-03-21 16:34:26 -07001465static bool mem_cgroup_stolen(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001466{
1467 VM_BUG_ON(!rcu_read_lock_held());
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001468 return atomic_read(&memcg->moving_account) > 0;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001469}
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001470
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001471static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001472{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001473 struct mem_cgroup *from;
1474 struct mem_cgroup *to;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001475 bool ret = false;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001476 /*
1477 * Unlike task_move routines, we access mc.to, mc.from not under
1478 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1479 */
1480 spin_lock(&mc.lock);
1481 from = mc.from;
1482 to = mc.to;
1483 if (!from)
1484 goto unlock;
Michal Hocko3e920412011-07-26 16:08:29 -07001485
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001486 ret = mem_cgroup_same_or_subtree(memcg, from)
1487 || mem_cgroup_same_or_subtree(memcg, to);
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001488unlock:
1489 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001490 return ret;
1491}
1492
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001493static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001494{
1495 if (mc.moving_task && current != mc.moving_task) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001496 if (mem_cgroup_under_move(memcg)) {
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001497 DEFINE_WAIT(wait);
1498 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1499 /* moving charge context might have finished. */
1500 if (mc.moving_task)
1501 schedule();
1502 finish_wait(&mc.waitq, &wait);
1503 return true;
1504 }
1505 }
1506 return false;
1507}
1508
KAMEZAWA Hiroyuki312734c02012-03-21 16:34:24 -07001509/*
1510 * Take this lock when
1511 * - a code tries to modify page's memcg while it's USED.
1512 * - a code tries to modify page state accounting in a memcg.
Andrew Morton13fd1dd92012-03-21 16:34:26 -07001513 * see mem_cgroup_stolen(), too.
KAMEZAWA Hiroyuki312734c02012-03-21 16:34:24 -07001514 */
1515static void move_lock_mem_cgroup(struct mem_cgroup *memcg,
1516 unsigned long *flags)
1517{
1518 spin_lock_irqsave(&memcg->move_lock, *flags);
1519}
1520
1521static void move_unlock_mem_cgroup(struct mem_cgroup *memcg,
1522 unsigned long *flags)
1523{
1524 spin_unlock_irqrestore(&memcg->move_lock, *flags);
1525}
1526
Balbir Singhe2224322009-04-02 16:57:39 -07001527/**
Kirill A. Shutemov6a6135b2010-03-10 15:22:25 -08001528 * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
Balbir Singhe2224322009-04-02 16:57:39 -07001529 * @memcg: The memory cgroup that went over limit
1530 * @p: Task that is going to be killed
1531 *
1532 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1533 * enabled
1534 */
1535void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1536{
1537 struct cgroup *task_cgrp;
1538 struct cgroup *mem_cgrp;
1539 /*
1540 * Need a buffer in BSS, can't rely on allocations. The code relies
1541 * on the assumption that OOM is serialized for memory controller.
1542 * If this assumption is broken, revisit this code.
1543 */
1544 static char memcg_name[PATH_MAX];
1545 int ret;
1546
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -08001547 if (!memcg || !p)
Balbir Singhe2224322009-04-02 16:57:39 -07001548 return;
1549
Balbir Singhe2224322009-04-02 16:57:39 -07001550 rcu_read_lock();
1551
1552 mem_cgrp = memcg->css.cgroup;
1553 task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1554
1555 ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1556 if (ret < 0) {
1557 /*
1558 * Unfortunately, we are unable to convert to a useful name
1559 * But we'll still print out the usage information
1560 */
1561 rcu_read_unlock();
1562 goto done;
1563 }
1564 rcu_read_unlock();
1565
1566 printk(KERN_INFO "Task in %s killed", memcg_name);
1567
1568 rcu_read_lock();
1569 ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1570 if (ret < 0) {
1571 rcu_read_unlock();
1572 goto done;
1573 }
1574 rcu_read_unlock();
1575
1576 /*
1577 * Continues from above, so we don't need an KERN_ level
1578 */
1579 printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
1580done:
1581
1582 printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
1583 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1584 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1585 res_counter_read_u64(&memcg->res, RES_FAILCNT));
1586 printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
1587 "failcnt %llu\n",
1588 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1589 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1590 res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
Glauber Costa510fc4e2012-12-18 14:21:47 -08001591 printk(KERN_INFO "kmem: usage %llukB, limit %llukB, failcnt %llu\n",
1592 res_counter_read_u64(&memcg->kmem, RES_USAGE) >> 10,
1593 res_counter_read_u64(&memcg->kmem, RES_LIMIT) >> 10,
1594 res_counter_read_u64(&memcg->kmem, RES_FAILCNT));
Balbir Singhe2224322009-04-02 16:57:39 -07001595}
1596
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001597/*
1598 * This function returns the number of memcg under hierarchy tree. Returns
1599 * 1(self count) if no children.
1600 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001601static int mem_cgroup_count_children(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001602{
1603 int num = 0;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001604 struct mem_cgroup *iter;
1605
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001606 for_each_mem_cgroup_tree(iter, memcg)
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001607 num++;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001608 return num;
1609}
1610
Balbir Singh6d61ef42009-01-07 18:08:06 -08001611/*
David Rientjesa63d83f2010-08-09 17:19:46 -07001612 * Return the memory (and swap, if configured) limit for a memcg.
1613 */
David Rientjes9cbb78b2012-07-31 16:43:44 -07001614static u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
David Rientjesa63d83f2010-08-09 17:19:46 -07001615{
1616 u64 limit;
David Rientjesa63d83f2010-08-09 17:19:46 -07001617
Johannes Weinerf3e8eb72011-01-13 15:47:39 -08001618 limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
Johannes Weinerf3e8eb72011-01-13 15:47:39 -08001619
David Rientjesa63d83f2010-08-09 17:19:46 -07001620 /*
Michal Hocko9a5a8f12012-11-16 14:14:49 -08001621 * Do not consider swap space if we cannot swap due to swappiness
David Rientjesa63d83f2010-08-09 17:19:46 -07001622 */
Michal Hocko9a5a8f12012-11-16 14:14:49 -08001623 if (mem_cgroup_swappiness(memcg)) {
1624 u64 memsw;
1625
1626 limit += total_swap_pages << PAGE_SHIFT;
1627 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1628
1629 /*
1630 * If memsw is finite and limits the amount of swap space
1631 * available to this memcg, return that limit.
1632 */
1633 limit = min(limit, memsw);
1634 }
1635
1636 return limit;
David Rientjesa63d83f2010-08-09 17:19:46 -07001637}
1638
David Rientjes19965462012-12-11 16:00:26 -08001639static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1640 int order)
David Rientjes9cbb78b2012-07-31 16:43:44 -07001641{
1642 struct mem_cgroup *iter;
1643 unsigned long chosen_points = 0;
1644 unsigned long totalpages;
1645 unsigned int points = 0;
1646 struct task_struct *chosen = NULL;
1647
David Rientjes876aafb2012-07-31 16:43:48 -07001648 /*
1649 * If current has a pending SIGKILL, then automatically select it. The
1650 * goal is to allow it to allocate so that it may quickly exit and free
1651 * its memory.
1652 */
1653 if (fatal_signal_pending(current)) {
1654 set_thread_flag(TIF_MEMDIE);
1655 return;
1656 }
1657
1658 check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL);
David Rientjes9cbb78b2012-07-31 16:43:44 -07001659 totalpages = mem_cgroup_get_limit(memcg) >> PAGE_SHIFT ? : 1;
1660 for_each_mem_cgroup_tree(iter, memcg) {
1661 struct cgroup *cgroup = iter->css.cgroup;
1662 struct cgroup_iter it;
1663 struct task_struct *task;
1664
1665 cgroup_iter_start(cgroup, &it);
1666 while ((task = cgroup_iter_next(cgroup, &it))) {
1667 switch (oom_scan_process_thread(task, totalpages, NULL,
1668 false)) {
1669 case OOM_SCAN_SELECT:
1670 if (chosen)
1671 put_task_struct(chosen);
1672 chosen = task;
1673 chosen_points = ULONG_MAX;
1674 get_task_struct(chosen);
1675 /* fall through */
1676 case OOM_SCAN_CONTINUE:
1677 continue;
1678 case OOM_SCAN_ABORT:
1679 cgroup_iter_end(cgroup, &it);
1680 mem_cgroup_iter_break(memcg, iter);
1681 if (chosen)
1682 put_task_struct(chosen);
1683 return;
1684 case OOM_SCAN_OK:
1685 break;
1686 };
1687 points = oom_badness(task, memcg, NULL, totalpages);
1688 if (points > chosen_points) {
1689 if (chosen)
1690 put_task_struct(chosen);
1691 chosen = task;
1692 chosen_points = points;
1693 get_task_struct(chosen);
1694 }
1695 }
1696 cgroup_iter_end(cgroup, &it);
1697 }
1698
1699 if (!chosen)
1700 return;
1701 points = chosen_points * 1000 / totalpages;
David Rientjes9cbb78b2012-07-31 16:43:44 -07001702 oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg,
1703 NULL, "Memory cgroup out of memory");
David Rientjes9cbb78b2012-07-31 16:43:44 -07001704}
1705
Johannes Weiner56600482012-01-12 17:17:59 -08001706static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
1707 gfp_t gfp_mask,
1708 unsigned long flags)
1709{
1710 unsigned long total = 0;
1711 bool noswap = false;
1712 int loop;
1713
1714 if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
1715 noswap = true;
1716 if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
1717 noswap = true;
1718
1719 for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
1720 if (loop)
1721 drain_all_stock_async(memcg);
1722 total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
1723 /*
1724 * Allow limit shrinkers, which are triggered directly
1725 * by userspace, to catch signals and stop reclaim
1726 * after minimal progress, regardless of the margin.
1727 */
1728 if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
1729 break;
1730 if (mem_cgroup_margin(memcg))
1731 break;
1732 /*
1733 * If nothing was reclaimed after two attempts, there
1734 * may be no reclaimable pages in this hierarchy.
1735 */
1736 if (loop && !total)
1737 break;
1738 }
1739 return total;
1740}
1741
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001742/**
1743 * test_mem_cgroup_node_reclaimable
Wanpeng Lidad75572012-06-20 12:53:01 -07001744 * @memcg: the target memcg
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001745 * @nid: the node ID to be checked.
1746 * @noswap : specify true here if the user wants flle only information.
1747 *
1748 * This function returns whether the specified memcg contains any
1749 * reclaimable pages on a node. Returns true if there are any reclaimable
1750 * pages in the node.
1751 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001752static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001753 int nid, bool noswap)
1754{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001755 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001756 return true;
1757 if (noswap || !total_swap_pages)
1758 return false;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001759 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001760 return true;
1761 return false;
1762
1763}
Ying Han889976d2011-05-26 16:25:33 -07001764#if MAX_NUMNODES > 1
1765
1766/*
1767 * Always updating the nodemask is not very good - even if we have an empty
1768 * list or the wrong list here, we can start from some node and traverse all
1769 * nodes based on the zonelist. So update the list loosely once per 10 secs.
1770 *
1771 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001772static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
Ying Han889976d2011-05-26 16:25:33 -07001773{
1774 int nid;
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -07001775 /*
1776 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1777 * pagein/pageout changes since the last update.
1778 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001779 if (!atomic_read(&memcg->numainfo_events))
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -07001780 return;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001781 if (atomic_inc_return(&memcg->numainfo_updating) > 1)
Ying Han889976d2011-05-26 16:25:33 -07001782 return;
1783
Ying Han889976d2011-05-26 16:25:33 -07001784 /* make a nodemask where this memcg uses memory from */
Lai Jiangshan31aaea42012-12-12 13:51:27 -08001785 memcg->scan_nodes = node_states[N_MEMORY];
Ying Han889976d2011-05-26 16:25:33 -07001786
Lai Jiangshan31aaea42012-12-12 13:51:27 -08001787 for_each_node_mask(nid, node_states[N_MEMORY]) {
Ying Han889976d2011-05-26 16:25:33 -07001788
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001789 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1790 node_clear(nid, memcg->scan_nodes);
Ying Han889976d2011-05-26 16:25:33 -07001791 }
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -07001792
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001793 atomic_set(&memcg->numainfo_events, 0);
1794 atomic_set(&memcg->numainfo_updating, 0);
Ying Han889976d2011-05-26 16:25:33 -07001795}
1796
1797/*
1798 * Selecting a node where we start reclaim from. Because what we need is just
1799 * reducing usage counter, start from anywhere is O,K. Considering
1800 * memory reclaim from current node, there are pros. and cons.
1801 *
1802 * Freeing memory from current node means freeing memory from a node which
1803 * we'll use or we've used. So, it may make LRU bad. And if several threads
1804 * hit limits, it will see a contention on a node. But freeing from remote
1805 * node means more costs for memory reclaim because of memory latency.
1806 *
1807 * Now, we use round-robin. Better algorithm is welcomed.
1808 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001809int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
Ying Han889976d2011-05-26 16:25:33 -07001810{
1811 int node;
1812
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001813 mem_cgroup_may_update_nodemask(memcg);
1814 node = memcg->last_scanned_node;
Ying Han889976d2011-05-26 16:25:33 -07001815
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001816 node = next_node(node, memcg->scan_nodes);
Ying Han889976d2011-05-26 16:25:33 -07001817 if (node == MAX_NUMNODES)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001818 node = first_node(memcg->scan_nodes);
Ying Han889976d2011-05-26 16:25:33 -07001819 /*
1820 * We call this when we hit limit, not when pages are added to LRU.
1821 * No LRU may hold pages because all pages are UNEVICTABLE or
1822 * memcg is too small and all pages are not on LRU. In that case,
1823 * we use curret node.
1824 */
1825 if (unlikely(node == MAX_NUMNODES))
1826 node = numa_node_id();
1827
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001828 memcg->last_scanned_node = node;
Ying Han889976d2011-05-26 16:25:33 -07001829 return node;
1830}
1831
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001832/*
1833 * Check all nodes whether it contains reclaimable pages or not.
1834 * For quick scan, we make use of scan_nodes. This will allow us to skip
1835 * unused nodes. But scan_nodes is lazily updated and may not cotain
1836 * enough new information. We need to do double check.
1837 */
Kirill A. Shutemov6bbda352012-05-29 15:06:55 -07001838static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001839{
1840 int nid;
1841
1842 /*
1843 * quick check...making use of scan_node.
1844 * We can skip unused nodes.
1845 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001846 if (!nodes_empty(memcg->scan_nodes)) {
1847 for (nid = first_node(memcg->scan_nodes);
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001848 nid < MAX_NUMNODES;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001849 nid = next_node(nid, memcg->scan_nodes)) {
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001850
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001851 if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001852 return true;
1853 }
1854 }
1855 /*
1856 * Check rest of nodes.
1857 */
Lai Jiangshan31aaea42012-12-12 13:51:27 -08001858 for_each_node_state(nid, N_MEMORY) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001859 if (node_isset(nid, memcg->scan_nodes))
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001860 continue;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001861 if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001862 return true;
1863 }
1864 return false;
1865}
1866
Ying Han889976d2011-05-26 16:25:33 -07001867#else
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001868int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
Ying Han889976d2011-05-26 16:25:33 -07001869{
1870 return 0;
1871}
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001872
Kirill A. Shutemov6bbda352012-05-29 15:06:55 -07001873static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001874{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001875 return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001876}
Ying Han889976d2011-05-26 16:25:33 -07001877#endif
1878
Johannes Weiner56600482012-01-12 17:17:59 -08001879static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1880 struct zone *zone,
1881 gfp_t gfp_mask,
1882 unsigned long *total_scanned)
Balbir Singh6d61ef42009-01-07 18:08:06 -08001883{
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001884 struct mem_cgroup *victim = NULL;
Johannes Weiner56600482012-01-12 17:17:59 -08001885 int total = 0;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001886 int loop = 0;
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001887 unsigned long excess;
Johannes Weiner185efc02011-09-14 16:21:58 -07001888 unsigned long nr_scanned;
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001889 struct mem_cgroup_reclaim_cookie reclaim = {
1890 .zone = zone,
1891 .priority = 0,
1892 };
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001893
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001894 excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001895
Balbir Singh4e416952009-09-23 15:56:39 -07001896 while (1) {
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001897 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001898 if (!victim) {
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001899 loop++;
Balbir Singh4e416952009-09-23 15:56:39 -07001900 if (loop >= 2) {
1901 /*
1902 * If we have not been able to reclaim
1903 * anything, it might because there are
1904 * no reclaimable pages under this hierarchy
1905 */
Johannes Weiner56600482012-01-12 17:17:59 -08001906 if (!total)
Balbir Singh4e416952009-09-23 15:56:39 -07001907 break;
Balbir Singh4e416952009-09-23 15:56:39 -07001908 /*
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001909 * We want to do more targeted reclaim.
Balbir Singh4e416952009-09-23 15:56:39 -07001910 * excess >> 2 is not to excessive so as to
1911 * reclaim too much, nor too less that we keep
1912 * coming back to reclaim from this cgroup
1913 */
1914 if (total >= (excess >> 2) ||
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001915 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
Balbir Singh4e416952009-09-23 15:56:39 -07001916 break;
Balbir Singh4e416952009-09-23 15:56:39 -07001917 }
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001918 continue;
Balbir Singh4e416952009-09-23 15:56:39 -07001919 }
Johannes Weiner56600482012-01-12 17:17:59 -08001920 if (!mem_cgroup_reclaimable(victim, false))
Balbir Singh6d61ef42009-01-07 18:08:06 -08001921 continue;
Johannes Weiner56600482012-01-12 17:17:59 -08001922 total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
1923 zone, &nr_scanned);
1924 *total_scanned += nr_scanned;
1925 if (!res_counter_soft_limit_excess(&root_memcg->res))
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001926 break;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001927 }
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001928 mem_cgroup_iter_break(root_memcg, victim);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07001929 return total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08001930}
1931
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001932/*
1933 * Check OOM-Killer is already running under our hierarchy.
1934 * If someone is running, return false.
Michal Hocko1af8efe2011-07-26 16:08:24 -07001935 * Has to be called with memcg_oom_lock
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001936 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001937static bool mem_cgroup_oom_lock(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001938{
Michal Hocko79dfdac2011-07-26 16:08:23 -07001939 struct mem_cgroup *iter, *failed = NULL;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001940
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001941 for_each_mem_cgroup_tree(iter, memcg) {
Johannes Weiner23751be2011-08-25 15:59:16 -07001942 if (iter->oom_lock) {
Michal Hocko79dfdac2011-07-26 16:08:23 -07001943 /*
1944 * this subtree of our hierarchy is already locked
1945 * so we cannot give a lock.
1946 */
Michal Hocko79dfdac2011-07-26 16:08:23 -07001947 failed = iter;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001948 mem_cgroup_iter_break(memcg, iter);
1949 break;
Johannes Weiner23751be2011-08-25 15:59:16 -07001950 } else
1951 iter->oom_lock = true;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001952 }
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001953
Michal Hocko79dfdac2011-07-26 16:08:23 -07001954 if (!failed)
Johannes Weiner23751be2011-08-25 15:59:16 -07001955 return true;
Michal Hocko79dfdac2011-07-26 16:08:23 -07001956
1957 /*
1958 * OK, we failed to lock the whole subtree so we have to clean up
1959 * what we set up to the failing subtree
1960 */
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001961 for_each_mem_cgroup_tree(iter, memcg) {
Michal Hocko79dfdac2011-07-26 16:08:23 -07001962 if (iter == failed) {
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001963 mem_cgroup_iter_break(memcg, iter);
1964 break;
Michal Hocko79dfdac2011-07-26 16:08:23 -07001965 }
1966 iter->oom_lock = false;
1967 }
Johannes Weiner23751be2011-08-25 15:59:16 -07001968 return false;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08001969}
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001970
Michal Hocko79dfdac2011-07-26 16:08:23 -07001971/*
Michal Hocko1af8efe2011-07-26 16:08:24 -07001972 * Has to be called with memcg_oom_lock
Michal Hocko79dfdac2011-07-26 16:08:23 -07001973 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001974static int mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001975{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001976 struct mem_cgroup *iter;
1977
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001978 for_each_mem_cgroup_tree(iter, memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001979 iter->oom_lock = false;
1980 return 0;
1981}
1982
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001983static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001984{
1985 struct mem_cgroup *iter;
1986
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001987 for_each_mem_cgroup_tree(iter, memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001988 atomic_inc(&iter->under_oom);
1989}
1990
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001991static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07001992{
1993 struct mem_cgroup *iter;
1994
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08001995 /*
1996 * When a new child is created while the hierarchy is under oom,
1997 * mem_cgroup_oom_lock() may not be called. We have to use
1998 * atomic_add_unless() here.
1999 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002000 for_each_mem_cgroup_tree(iter, memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07002001 atomic_add_unless(&iter->under_oom, -1, 0);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07002002}
2003
Michal Hocko1af8efe2011-07-26 16:08:24 -07002004static DEFINE_SPINLOCK(memcg_oom_lock);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002005static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
2006
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002007struct oom_wait_info {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07002008 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002009 wait_queue_t wait;
2010};
2011
2012static int memcg_oom_wake_function(wait_queue_t *wait,
2013 unsigned mode, int sync, void *arg)
2014{
Hugh Dickinsd79154b2012-03-21 16:34:18 -07002015 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
2016 struct mem_cgroup *oom_wait_memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002017 struct oom_wait_info *oom_wait_info;
2018
2019 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
Hugh Dickinsd79154b2012-03-21 16:34:18 -07002020 oom_wait_memcg = oom_wait_info->memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002021
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002022 /*
Hugh Dickinsd79154b2012-03-21 16:34:18 -07002023 * Both of oom_wait_info->memcg and wake_memcg are stable under us.
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002024 * Then we can use css_is_ancestor without taking care of RCU.
2025 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002026 if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
2027 && !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002028 return 0;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002029 return autoremove_wake_function(wait, mode, sync, arg);
2030}
2031
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002032static void memcg_wakeup_oom(struct mem_cgroup *memcg)
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002033{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002034 /* for filtering, pass "memcg" as argument. */
2035 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002036}
2037
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002038static void memcg_oom_recover(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002039{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002040 if (memcg && atomic_read(&memcg->under_oom))
2041 memcg_wakeup_oom(memcg);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002042}
2043
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002044/*
2045 * try to call OOM killer. returns false if we should exit memory-reclaim loop.
2046 */
Kirill A. Shutemov6bbda352012-05-29 15:06:55 -07002047static bool mem_cgroup_handle_oom(struct mem_cgroup *memcg, gfp_t mask,
2048 int order)
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002049{
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002050 struct oom_wait_info owait;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002051 bool locked, need_to_kill;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002052
Hugh Dickinsd79154b2012-03-21 16:34:18 -07002053 owait.memcg = memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002054 owait.wait.flags = 0;
2055 owait.wait.func = memcg_oom_wake_function;
2056 owait.wait.private = current;
2057 INIT_LIST_HEAD(&owait.wait.task_list);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002058 need_to_kill = true;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002059 mem_cgroup_mark_under_oom(memcg);
Michal Hocko79dfdac2011-07-26 16:08:23 -07002060
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002061 /* At first, try to OOM lock hierarchy under memcg.*/
Michal Hocko1af8efe2011-07-26 16:08:24 -07002062 spin_lock(&memcg_oom_lock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002063 locked = mem_cgroup_oom_lock(memcg);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002064 /*
2065 * Even if signal_pending(), we can't quit charge() loop without
2066 * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
2067 * under OOM is always welcomed, use TASK_KILLABLE here.
2068 */
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002069 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002070 if (!locked || memcg->oom_kill_disable)
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002071 need_to_kill = false;
2072 if (locked)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002073 mem_cgroup_oom_notify(memcg);
Michal Hocko1af8efe2011-07-26 16:08:24 -07002074 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002075
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002076 if (need_to_kill) {
2077 finish_wait(&memcg_oom_waitq, &owait.wait);
David Rientjese845e192012-03-21 16:34:10 -07002078 mem_cgroup_out_of_memory(memcg, mask, order);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002079 } else {
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002080 schedule();
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002081 finish_wait(&memcg_oom_waitq, &owait.wait);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002082 }
Michal Hocko1af8efe2011-07-26 16:08:24 -07002083 spin_lock(&memcg_oom_lock);
Michal Hocko79dfdac2011-07-26 16:08:23 -07002084 if (locked)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002085 mem_cgroup_oom_unlock(memcg);
2086 memcg_wakeup_oom(memcg);
Michal Hocko1af8efe2011-07-26 16:08:24 -07002087 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002088
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002089 mem_cgroup_unmark_under_oom(memcg);
Michal Hocko79dfdac2011-07-26 16:08:23 -07002090
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002091 if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
2092 return false;
2093 /* Give chance to dying process */
KAMEZAWA Hiroyuki715a5ee2011-11-02 13:38:18 -07002094 schedule_timeout_uninterruptible(1);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002095 return true;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07002096}
2097
Balbir Singhd69b0422009-06-17 16:26:34 -07002098/*
2099 * Currently used to update mapped file statistics, but the routine can be
2100 * generalized to update other statistics as well.
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002101 *
2102 * Notes: Race condition
2103 *
2104 * We usually use page_cgroup_lock() for accessing page_cgroup member but
2105 * it tends to be costly. But considering some conditions, we doesn't need
2106 * to do so _always_.
2107 *
2108 * Considering "charge", lock_page_cgroup() is not required because all
2109 * file-stat operations happen after a page is attached to radix-tree. There
2110 * are no race with "charge".
2111 *
2112 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
2113 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
2114 * if there are race with "uncharge". Statistics itself is properly handled
2115 * by flags.
2116 *
2117 * Considering "move", this is an only case we see a race. To make the race
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07002118 * small, we check mm->moving_account and detect there are possibility of race
2119 * If there is, we take a lock.
Balbir Singhd69b0422009-06-17 16:26:34 -07002120 */
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002121
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002122void __mem_cgroup_begin_update_page_stat(struct page *page,
2123 bool *locked, unsigned long *flags)
2124{
2125 struct mem_cgroup *memcg;
2126 struct page_cgroup *pc;
2127
2128 pc = lookup_page_cgroup(page);
2129again:
2130 memcg = pc->mem_cgroup;
2131 if (unlikely(!memcg || !PageCgroupUsed(pc)))
2132 return;
2133 /*
2134 * If this memory cgroup is not under account moving, we don't
Wanpeng Lida92c472012-07-31 16:43:26 -07002135 * need to take move_lock_mem_cgroup(). Because we already hold
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002136 * rcu_read_lock(), any calls to move_account will be delayed until
Andrew Morton13fd1dd92012-03-21 16:34:26 -07002137 * rcu_read_unlock() if mem_cgroup_stolen() == true.
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002138 */
Andrew Morton13fd1dd92012-03-21 16:34:26 -07002139 if (!mem_cgroup_stolen(memcg))
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002140 return;
2141
2142 move_lock_mem_cgroup(memcg, flags);
2143 if (memcg != pc->mem_cgroup || !PageCgroupUsed(pc)) {
2144 move_unlock_mem_cgroup(memcg, flags);
2145 goto again;
2146 }
2147 *locked = true;
2148}
2149
2150void __mem_cgroup_end_update_page_stat(struct page *page, unsigned long *flags)
2151{
2152 struct page_cgroup *pc = lookup_page_cgroup(page);
2153
2154 /*
2155 * It's guaranteed that pc->mem_cgroup never changes while
2156 * lock is held because a routine modifies pc->mem_cgroup
Wanpeng Lida92c472012-07-31 16:43:26 -07002157 * should take move_lock_mem_cgroup().
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002158 */
2159 move_unlock_mem_cgroup(pc->mem_cgroup, flags);
2160}
2161
Greg Thelen2a7106f2011-01-13 15:47:37 -08002162void mem_cgroup_update_page_stat(struct page *page,
2163 enum mem_cgroup_page_stat_item idx, int val)
Balbir Singhd69b0422009-06-17 16:26:34 -07002164{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002165 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002166 struct page_cgroup *pc = lookup_page_cgroup(page);
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08002167 unsigned long uninitialized_var(flags);
Balbir Singhd69b0422009-06-17 16:26:34 -07002168
Johannes Weinercfa44942012-01-12 17:18:38 -08002169 if (mem_cgroup_disabled())
Balbir Singhd69b0422009-06-17 16:26:34 -07002170 return;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002171
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002172 memcg = pc->mem_cgroup;
2173 if (unlikely(!memcg || !PageCgroupUsed(pc)))
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002174 return;
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002175
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002176 switch (idx) {
Greg Thelen2a7106f2011-01-13 15:47:37 -08002177 case MEMCG_NR_FILE_MAPPED:
Greg Thelen2a7106f2011-01-13 15:47:37 -08002178 idx = MEM_CGROUP_STAT_FILE_MAPPED;
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002179 break;
2180 default:
2181 BUG();
KAMEZAWA Hiroyuki8725d542010-04-06 14:35:05 -07002182 }
Balbir Singhd69b0422009-06-17 16:26:34 -07002183
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002184 this_cpu_add(memcg->stat->count[idx], val);
Balbir Singhd69b0422009-06-17 16:26:34 -07002185}
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002186
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002187/*
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002188 * size of first charge trial. "32" comes from vmscan.c's magic value.
2189 * TODO: maybe necessary to use big numbers in big irons.
2190 */
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002191#define CHARGE_BATCH 32U
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002192struct memcg_stock_pcp {
2193 struct mem_cgroup *cached; /* this never be root cgroup */
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002194 unsigned int nr_pages;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002195 struct work_struct work;
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002196 unsigned long flags;
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -07002197#define FLUSHING_CACHED_CHARGE 0
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002198};
2199static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
Michal Hocko9f50fad2011-08-09 11:56:26 +02002200static DEFINE_MUTEX(percpu_charge_mutex);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002201
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002202/**
2203 * consume_stock: Try to consume stocked charge on this cpu.
2204 * @memcg: memcg to consume from.
2205 * @nr_pages: how many pages to charge.
2206 *
2207 * The charges will only happen if @memcg matches the current cpu's memcg
2208 * stock, and at least @nr_pages are available in that stock. Failure to
2209 * service an allocation will refill the stock.
2210 *
2211 * returns true if successful, false otherwise.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002212 */
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002213static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002214{
2215 struct memcg_stock_pcp *stock;
2216 bool ret = true;
2217
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002218 if (nr_pages > CHARGE_BATCH)
2219 return false;
2220
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002221 stock = &get_cpu_var(memcg_stock);
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002222 if (memcg == stock->cached && stock->nr_pages >= nr_pages)
2223 stock->nr_pages -= nr_pages;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002224 else /* need to call res_counter_charge */
2225 ret = false;
2226 put_cpu_var(memcg_stock);
2227 return ret;
2228}
2229
2230/*
2231 * Returns stocks cached in percpu to res_counter and reset cached information.
2232 */
2233static void drain_stock(struct memcg_stock_pcp *stock)
2234{
2235 struct mem_cgroup *old = stock->cached;
2236
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002237 if (stock->nr_pages) {
2238 unsigned long bytes = stock->nr_pages * PAGE_SIZE;
2239
2240 res_counter_uncharge(&old->res, bytes);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002241 if (do_swap_account)
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002242 res_counter_uncharge(&old->memsw, bytes);
2243 stock->nr_pages = 0;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002244 }
2245 stock->cached = NULL;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002246}
2247
2248/*
2249 * This must be called under preempt disabled or must be called by
2250 * a thread which is pinned to local cpu.
2251 */
2252static void drain_local_stock(struct work_struct *dummy)
2253{
2254 struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
2255 drain_stock(stock);
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002256 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002257}
2258
2259/*
2260 * Cache charges(val) which is from res_counter, to local per_cpu area.
Greg Thelen320cc512010-03-15 15:27:28 +01002261 * This will be consumed by consume_stock() function, later.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002262 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002263static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002264{
2265 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
2266
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002267 if (stock->cached != memcg) { /* reset if necessary */
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002268 drain_stock(stock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002269 stock->cached = memcg;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002270 }
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002271 stock->nr_pages += nr_pages;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002272 put_cpu_var(memcg_stock);
2273}
2274
2275/*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002276 * Drains all per-CPU charge caches for given root_memcg resp. subtree
Michal Hockod38144b2011-07-26 16:08:28 -07002277 * of the hierarchy under it. sync flag says whether we should block
2278 * until the work is done.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002279 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002280static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002281{
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002282 int cpu, curcpu;
Michal Hockod38144b2011-07-26 16:08:28 -07002283
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002284 /* Notify other cpus that system-wide "drain" is running */
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002285 get_online_cpus();
Johannes Weiner5af12d02011-08-25 15:59:07 -07002286 curcpu = get_cpu();
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002287 for_each_online_cpu(cpu) {
2288 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002289 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002290
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002291 memcg = stock->cached;
2292 if (!memcg || !stock->nr_pages)
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002293 continue;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002294 if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
Michal Hocko3e920412011-07-26 16:08:29 -07002295 continue;
Michal Hockod1a05b62011-07-26 16:08:27 -07002296 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2297 if (cpu == curcpu)
2298 drain_local_stock(&stock->work);
2299 else
2300 schedule_work_on(cpu, &stock->work);
2301 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002302 }
Johannes Weiner5af12d02011-08-25 15:59:07 -07002303 put_cpu();
Michal Hockod38144b2011-07-26 16:08:28 -07002304
2305 if (!sync)
2306 goto out;
2307
2308 for_each_online_cpu(cpu) {
2309 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
Michal Hocko9f50fad2011-08-09 11:56:26 +02002310 if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
Michal Hockod38144b2011-07-26 16:08:28 -07002311 flush_work(&stock->work);
2312 }
2313out:
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002314 put_online_cpus();
Michal Hockod38144b2011-07-26 16:08:28 -07002315}
2316
2317/*
2318 * Tries to drain stocked charges in other cpus. This function is asynchronous
2319 * and just put a work per cpu for draining localy on each cpu. Caller can
2320 * expects some charges will be back to res_counter later but cannot wait for
2321 * it.
2322 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002323static void drain_all_stock_async(struct mem_cgroup *root_memcg)
Michal Hockod38144b2011-07-26 16:08:28 -07002324{
Michal Hocko9f50fad2011-08-09 11:56:26 +02002325 /*
2326 * If someone calls draining, avoid adding more kworker runs.
2327 */
2328 if (!mutex_trylock(&percpu_charge_mutex))
2329 return;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002330 drain_all_stock(root_memcg, false);
Michal Hocko9f50fad2011-08-09 11:56:26 +02002331 mutex_unlock(&percpu_charge_mutex);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002332}
2333
2334/* This is a synchronous drain interface. */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002335static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002336{
2337 /* called when force_empty is called */
Michal Hocko9f50fad2011-08-09 11:56:26 +02002338 mutex_lock(&percpu_charge_mutex);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002339 drain_all_stock(root_memcg, true);
Michal Hocko9f50fad2011-08-09 11:56:26 +02002340 mutex_unlock(&percpu_charge_mutex);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002341}
2342
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002343/*
2344 * This function drains percpu counter value from DEAD cpu and
2345 * move it to local cpu. Note that this function can be preempted.
2346 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002347static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002348{
2349 int i;
2350
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002351 spin_lock(&memcg->pcp_counter_lock);
Johannes Weiner61046212012-05-29 15:07:05 -07002352 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002353 long x = per_cpu(memcg->stat->count[i], cpu);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002354
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002355 per_cpu(memcg->stat->count[i], cpu) = 0;
2356 memcg->nocpu_base.count[i] += x;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002357 }
Johannes Weinere9f89742011-03-23 16:42:37 -07002358 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002359 unsigned long x = per_cpu(memcg->stat->events[i], cpu);
Johannes Weinere9f89742011-03-23 16:42:37 -07002360
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002361 per_cpu(memcg->stat->events[i], cpu) = 0;
2362 memcg->nocpu_base.events[i] += x;
Johannes Weinere9f89742011-03-23 16:42:37 -07002363 }
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002364 spin_unlock(&memcg->pcp_counter_lock);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002365}
2366
2367static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002368 unsigned long action,
2369 void *hcpu)
2370{
2371 int cpu = (unsigned long)hcpu;
2372 struct memcg_stock_pcp *stock;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002373 struct mem_cgroup *iter;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002374
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07002375 if (action == CPU_ONLINE)
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07002376 return NOTIFY_OK;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07002377
Kirill A. Shutemovd8330492012-04-12 12:49:11 -07002378 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002379 return NOTIFY_OK;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002380
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08002381 for_each_mem_cgroup(iter)
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002382 mem_cgroup_drain_pcp_counter(iter, cpu);
2383
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002384 stock = &per_cpu(memcg_stock, cpu);
2385 drain_stock(stock);
2386 return NOTIFY_OK;
2387}
2388
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002389
2390/* See __mem_cgroup_try_charge() for details */
2391enum {
2392 CHARGE_OK, /* success */
2393 CHARGE_RETRY, /* need to retry but retry is not bad */
2394 CHARGE_NOMEM, /* we can't do more. return -ENOMEM */
2395 CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */
2396 CHARGE_OOM_DIE, /* the current is killed because of OOM */
2397};
2398
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002399static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
Suleiman Souhlal4c9c5352012-12-18 14:21:41 -08002400 unsigned int nr_pages, unsigned int min_pages,
2401 bool oom_check)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002402{
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002403 unsigned long csize = nr_pages * PAGE_SIZE;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002404 struct mem_cgroup *mem_over_limit;
2405 struct res_counter *fail_res;
2406 unsigned long flags = 0;
2407 int ret;
2408
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002409 ret = res_counter_charge(&memcg->res, csize, &fail_res);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002410
2411 if (likely(!ret)) {
2412 if (!do_swap_account)
2413 return CHARGE_OK;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002414 ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002415 if (likely(!ret))
2416 return CHARGE_OK;
2417
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002418 res_counter_uncharge(&memcg->res, csize);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002419 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
2420 flags |= MEM_CGROUP_RECLAIM_NOSWAP;
2421 } else
2422 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
Johannes Weiner9221edb2011-02-01 15:52:42 -08002423 /*
Johannes Weiner9221edb2011-02-01 15:52:42 -08002424 * Never reclaim on behalf of optional batching, retry with a
2425 * single page instead.
2426 */
Suleiman Souhlal4c9c5352012-12-18 14:21:41 -08002427 if (nr_pages > min_pages)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002428 return CHARGE_RETRY;
2429
2430 if (!(gfp_mask & __GFP_WAIT))
2431 return CHARGE_WOULDBLOCK;
2432
Suleiman Souhlal4c9c5352012-12-18 14:21:41 -08002433 if (gfp_mask & __GFP_NORETRY)
2434 return CHARGE_NOMEM;
2435
Johannes Weiner56600482012-01-12 17:17:59 -08002436 ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002437 if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
Johannes Weiner19942822011-02-01 15:52:43 -08002438 return CHARGE_RETRY;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002439 /*
Johannes Weiner19942822011-02-01 15:52:43 -08002440 * Even though the limit is exceeded at this point, reclaim
2441 * may have been able to free some pages. Retry the charge
2442 * before killing the task.
2443 *
2444 * Only for regular pages, though: huge pages are rather
2445 * unlikely to succeed so close to the limit, and we fall back
2446 * to regular pages anyway in case of failure.
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002447 */
Suleiman Souhlal4c9c5352012-12-18 14:21:41 -08002448 if (nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER) && ret)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002449 return CHARGE_RETRY;
2450
2451 /*
2452 * At task move, charge accounts can be doubly counted. So, it's
2453 * better to wait until the end of task_move if something is going on.
2454 */
2455 if (mem_cgroup_wait_acct_move(mem_over_limit))
2456 return CHARGE_RETRY;
2457
2458 /* If we don't need to call oom-killer at el, return immediately */
2459 if (!oom_check)
2460 return CHARGE_NOMEM;
2461 /* check OOM */
David Rientjese845e192012-03-21 16:34:10 -07002462 if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask, get_order(csize)))
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002463 return CHARGE_OOM_DIE;
2464
2465 return CHARGE_RETRY;
2466}
2467
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002468/*
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08002469 * __mem_cgroup_try_charge() does
2470 * 1. detect memcg to be charged against from passed *mm and *ptr,
2471 * 2. update res_counter
2472 * 3. call memory reclaim if necessary.
2473 *
2474 * In some special case, if the task is fatal, fatal_signal_pending() or
2475 * has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
2476 * to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
2477 * as possible without any hazards. 2: all pages should have a valid
2478 * pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
2479 * pointer, that is treated as a charge to root_mem_cgroup.
2480 *
2481 * So __mem_cgroup_try_charge() will return
2482 * 0 ... on success, filling *ptr with a valid memcg pointer.
2483 * -ENOMEM ... charge failure because of resource limits.
2484 * -EINTR ... if thread is fatal. *ptr is filled with root_mem_cgroup.
2485 *
2486 * Unlike the exported interface, an "oom" parameter is added. if oom==true,
2487 * the oom-killer can be invoked.
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002488 */
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002489static int __mem_cgroup_try_charge(struct mm_struct *mm,
Andrea Arcangeliec168512011-01-13 15:46:56 -08002490 gfp_t gfp_mask,
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002491 unsigned int nr_pages,
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002492 struct mem_cgroup **ptr,
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002493 bool oom)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002494{
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002495 unsigned int batch = max(CHARGE_BATCH, nr_pages);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002496 int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002497 struct mem_cgroup *memcg = NULL;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002498 int ret;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002499
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002500 /*
2501 * Unlike gloval-vm's OOM-kill, we're not in memory shortage
2502 * in system level. So, allow to go ahead dying process in addition to
2503 * MEMDIE process.
2504 */
2505 if (unlikely(test_thread_flag(TIF_MEMDIE)
2506 || fatal_signal_pending(current)))
2507 goto bypass;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002508
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002509 /*
Hugh Dickins3be91272008-02-07 00:14:19 -08002510 * We always charge the cgroup the mm_struct belongs to.
2511 * The mm_struct's mem_cgroup changes on task migration if the
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002512 * thread group leader migrates. It's possible that mm is not
Johannes Weiner24467ca2012-07-31 16:45:40 -07002513 * set, if so charge the root memcg (happens for pagecache usage).
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002514 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002515 if (!*ptr && !mm)
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08002516 *ptr = root_mem_cgroup;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002517again:
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002518 if (*ptr) { /* css should be a valid one */
2519 memcg = *ptr;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002520 if (mem_cgroup_is_root(memcg))
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002521 goto done;
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002522 if (consume_stock(memcg, nr_pages))
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002523 goto done;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002524 css_get(&memcg->css);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002525 } else {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002526 struct task_struct *p;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002527
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002528 rcu_read_lock();
2529 p = rcu_dereference(mm->owner);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002530 /*
KAMEZAWA Hiroyukiebb76ce2010-12-29 14:07:11 -08002531 * Because we don't have task_lock(), "p" can exit.
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002532 * In that case, "memcg" can point to root or p can be NULL with
KAMEZAWA Hiroyukiebb76ce2010-12-29 14:07:11 -08002533 * race with swapoff. Then, we have small risk of mis-accouning.
2534 * But such kind of mis-account by race always happens because
2535 * we don't have cgroup_mutex(). It's overkill and we allo that
2536 * small race, here.
2537 * (*) swapoff at el will charge against mm-struct not against
2538 * task-struct. So, mm->owner can be NULL.
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002539 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002540 memcg = mem_cgroup_from_task(p);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08002541 if (!memcg)
2542 memcg = root_mem_cgroup;
2543 if (mem_cgroup_is_root(memcg)) {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002544 rcu_read_unlock();
2545 goto done;
2546 }
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002547 if (consume_stock(memcg, nr_pages)) {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002548 /*
2549 * It seems dagerous to access memcg without css_get().
2550 * But considering how consume_stok works, it's not
2551 * necessary. If consume_stock success, some charges
2552 * from this memcg are cached on this cpu. So, we
2553 * don't need to call css_get()/css_tryget() before
2554 * calling consume_stock().
2555 */
2556 rcu_read_unlock();
2557 goto done;
2558 }
2559 /* after here, we may be blocked. we need to get refcnt */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002560 if (!css_tryget(&memcg->css)) {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002561 rcu_read_unlock();
2562 goto again;
2563 }
2564 rcu_read_unlock();
2565 }
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002566
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002567 do {
2568 bool oom_check;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002569
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002570 /* If killed, bypass charge */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002571 if (fatal_signal_pending(current)) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002572 css_put(&memcg->css);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002573 goto bypass;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002574 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002575
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002576 oom_check = false;
2577 if (oom && !nr_oom_retries) {
2578 oom_check = true;
2579 nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
2580 }
Balbir Singh6d61ef42009-01-07 18:08:06 -08002581
Suleiman Souhlal4c9c5352012-12-18 14:21:41 -08002582 ret = mem_cgroup_do_charge(memcg, gfp_mask, batch, nr_pages,
2583 oom_check);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002584 switch (ret) {
2585 case CHARGE_OK:
2586 break;
2587 case CHARGE_RETRY: /* not in OOM situation but retry */
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002588 batch = nr_pages;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002589 css_put(&memcg->css);
2590 memcg = NULL;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002591 goto again;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002592 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002593 css_put(&memcg->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002594 goto nomem;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002595 case CHARGE_NOMEM: /* OOM routine works */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002596 if (!oom) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002597 css_put(&memcg->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002598 goto nomem;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002599 }
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002600 /* If oom, we never return -ENOMEM */
2601 nr_oom_retries--;
2602 break;
2603 case CHARGE_OOM_DIE: /* Killed by OOM Killer */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002604 css_put(&memcg->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002605 goto bypass;
Balbir Singh66e17072008-02-07 00:13:56 -08002606 }
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002607 } while (ret != CHARGE_OK);
2608
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002609 if (batch > nr_pages)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002610 refill_stock(memcg, batch - nr_pages);
2611 css_put(&memcg->css);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002612done:
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002613 *ptr = memcg;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002614 return 0;
2615nomem:
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002616 *ptr = NULL;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002617 return -ENOMEM;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002618bypass:
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08002619 *ptr = root_mem_cgroup;
2620 return -EINTR;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002621}
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002622
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002623/*
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002624 * Somemtimes we have to undo a charge we got by try_charge().
2625 * This function is for that and do uncharge, put css's refcnt.
2626 * gotten by try_charge().
2627 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002628static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
Johannes Weinere7018b8d2011-03-23 16:42:33 -07002629 unsigned int nr_pages)
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002630{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002631 if (!mem_cgroup_is_root(memcg)) {
Johannes Weinere7018b8d2011-03-23 16:42:33 -07002632 unsigned long bytes = nr_pages * PAGE_SIZE;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002633
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002634 res_counter_uncharge(&memcg->res, bytes);
Johannes Weinere7018b8d2011-03-23 16:42:33 -07002635 if (do_swap_account)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002636 res_counter_uncharge(&memcg->memsw, bytes);
Johannes Weinere7018b8d2011-03-23 16:42:33 -07002637 }
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002638}
2639
2640/*
KAMEZAWA Hiroyukid01dd172012-05-29 15:07:03 -07002641 * Cancel chrages in this cgroup....doesn't propagate to parent cgroup.
2642 * This is useful when moving usage to parent cgroup.
2643 */
2644static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg,
2645 unsigned int nr_pages)
2646{
2647 unsigned long bytes = nr_pages * PAGE_SIZE;
2648
2649 if (mem_cgroup_is_root(memcg))
2650 return;
2651
2652 res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes);
2653 if (do_swap_account)
2654 res_counter_uncharge_until(&memcg->memsw,
2655 memcg->memsw.parent, bytes);
2656}
2657
2658/*
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002659 * A helper function to get mem_cgroup from ID. must be called under
Tejun Heoe9316082012-11-05 09:16:58 -08002660 * rcu_read_lock(). The caller is responsible for calling css_tryget if
2661 * the mem_cgroup is used for charging. (dropping refcnt from swap can be
2662 * called against removed memcg.)
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002663 */
2664static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
2665{
2666 struct cgroup_subsys_state *css;
2667
2668 /* ID 0 is unused ID */
2669 if (!id)
2670 return NULL;
2671 css = css_lookup(&mem_cgroup_subsys, id);
2672 if (!css)
2673 return NULL;
Wanpeng Lib2145142012-07-31 16:46:01 -07002674 return mem_cgroup_from_css(css);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002675}
2676
Wu Fengguange42d9d52009-12-16 12:19:59 +01002677struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002678{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002679 struct mem_cgroup *memcg = NULL;
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002680 struct page_cgroup *pc;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002681 unsigned short id;
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002682 swp_entry_t ent;
2683
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002684 VM_BUG_ON(!PageLocked(page));
2685
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002686 pc = lookup_page_cgroup(page);
Daisuke Nishimurac0bd3f62009-04-30 15:08:11 -07002687 lock_page_cgroup(pc);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002688 if (PageCgroupUsed(pc)) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002689 memcg = pc->mem_cgroup;
2690 if (memcg && !css_tryget(&memcg->css))
2691 memcg = NULL;
Wu Fengguange42d9d52009-12-16 12:19:59 +01002692 } else if (PageSwapCache(page)) {
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002693 ent.val = page_private(page);
Bob Liu9fb4b7c2012-01-12 17:18:48 -08002694 id = lookup_swap_cgroup_id(ent);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002695 rcu_read_lock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002696 memcg = mem_cgroup_lookup(id);
2697 if (memcg && !css_tryget(&memcg->css))
2698 memcg = NULL;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002699 rcu_read_unlock();
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002700 }
Daisuke Nishimurac0bd3f62009-04-30 15:08:11 -07002701 unlock_page_cgroup(pc);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002702 return memcg;
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002703}
2704
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002705static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
Johannes Weiner5564e882011-03-23 16:42:29 -07002706 struct page *page,
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002707 unsigned int nr_pages,
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002708 enum charge_type ctype,
2709 bool lrucare)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002710{
Johannes Weinerce587e62012-04-24 20:22:33 +02002711 struct page_cgroup *pc = lookup_page_cgroup(page);
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002712 struct zone *uninitialized_var(zone);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07002713 struct lruvec *lruvec;
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002714 bool was_on_lru = false;
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07002715 bool anon;
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002716
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002717 lock_page_cgroup(pc);
Johannes Weiner90deb782012-07-31 16:45:47 -07002718 VM_BUG_ON(PageCgroupUsed(pc));
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002719 /*
2720 * we don't need page_cgroup_lock about tail pages, becase they are not
2721 * accessed by any other context at this point.
2722 */
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002723
2724 /*
2725 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2726 * may already be on some other mem_cgroup's LRU. Take care of it.
2727 */
2728 if (lrucare) {
2729 zone = page_zone(page);
2730 spin_lock_irq(&zone->lru_lock);
2731 if (PageLRU(page)) {
Hugh Dickinsfa9add62012-05-29 15:07:09 -07002732 lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002733 ClearPageLRU(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07002734 del_page_from_lru_list(page, lruvec, page_lru(page));
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002735 was_on_lru = true;
2736 }
2737 }
2738
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002739 pc->mem_cgroup = memcg;
KAMEZAWA Hiroyuki261fb612009-09-23 15:56:33 -07002740 /*
2741 * We access a page_cgroup asynchronously without lock_page_cgroup().
2742 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
2743 * is accessed after testing USED bit. To make pc->mem_cgroup visible
2744 * before USED bit, we need memory barrier here.
2745 * See mem_cgroup_add_lru_list(), etc.
2746 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002747 smp_wmb();
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07002748 SetPageCgroupUsed(pc);
Hugh Dickins3be91272008-02-07 00:14:19 -08002749
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002750 if (lrucare) {
2751 if (was_on_lru) {
Hugh Dickinsfa9add62012-05-29 15:07:09 -07002752 lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002753 VM_BUG_ON(PageLRU(page));
2754 SetPageLRU(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07002755 add_page_to_lru_list(page, lruvec, page_lru(page));
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002756 }
2757 spin_unlock_irq(&zone->lru_lock);
2758 }
2759
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -07002760 if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON)
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07002761 anon = true;
2762 else
2763 anon = false;
2764
2765 mem_cgroup_charge_statistics(memcg, anon, nr_pages);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002766 unlock_page_cgroup(pc);
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002767
KAMEZAWA Hiroyuki430e48632010-03-10 15:22:30 -08002768 /*
2769 * "charge_statistics" updated event counter. Then, check it.
2770 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
2771 * if they exceeds softlimit.
2772 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002773 memcg_check_events(memcg, page);
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002774}
2775
Glauber Costa7cf27982012-12-18 14:22:55 -08002776static DEFINE_MUTEX(set_limit_mutex);
2777
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002778#ifdef CONFIG_MEMCG_KMEM
2779static inline bool memcg_can_account_kmem(struct mem_cgroup *memcg)
2780{
2781 return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg) &&
2782 (memcg->kmem_account_flags & KMEM_ACCOUNTED_MASK);
2783}
2784
Glauber Costa1f458cb2012-12-18 14:22:50 -08002785/*
2786 * This is a bit cumbersome, but it is rarely used and avoids a backpointer
2787 * in the memcg_cache_params struct.
2788 */
2789static struct kmem_cache *memcg_params_to_cache(struct memcg_cache_params *p)
2790{
2791 struct kmem_cache *cachep;
2792
2793 VM_BUG_ON(p->is_root_cache);
2794 cachep = p->root_cache;
2795 return cachep->memcg_params->memcg_caches[memcg_cache_id(p->memcg)];
2796}
2797
Glauber Costa749c5412012-12-18 14:23:01 -08002798#ifdef CONFIG_SLABINFO
2799static int mem_cgroup_slabinfo_read(struct cgroup *cont, struct cftype *cft,
2800 struct seq_file *m)
2801{
2802 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
2803 struct memcg_cache_params *params;
2804
2805 if (!memcg_can_account_kmem(memcg))
2806 return -EIO;
2807
2808 print_slabinfo_header(m);
2809
2810 mutex_lock(&memcg->slab_caches_mutex);
2811 list_for_each_entry(params, &memcg->memcg_slab_caches, list)
2812 cache_show(memcg_params_to_cache(params), m);
2813 mutex_unlock(&memcg->slab_caches_mutex);
2814
2815 return 0;
2816}
2817#endif
2818
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002819static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
2820{
2821 struct res_counter *fail_res;
2822 struct mem_cgroup *_memcg;
2823 int ret = 0;
2824 bool may_oom;
2825
2826 ret = res_counter_charge(&memcg->kmem, size, &fail_res);
2827 if (ret)
2828 return ret;
2829
2830 /*
2831 * Conditions under which we can wait for the oom_killer. Those are
2832 * the same conditions tested by the core page allocator
2833 */
2834 may_oom = (gfp & __GFP_FS) && !(gfp & __GFP_NORETRY);
2835
2836 _memcg = memcg;
2837 ret = __mem_cgroup_try_charge(NULL, gfp, size >> PAGE_SHIFT,
2838 &_memcg, may_oom);
2839
2840 if (ret == -EINTR) {
2841 /*
2842 * __mem_cgroup_try_charge() chosed to bypass to root due to
2843 * OOM kill or fatal signal. Since our only options are to
2844 * either fail the allocation or charge it to this cgroup, do
2845 * it as a temporary condition. But we can't fail. From a
2846 * kmem/slab perspective, the cache has already been selected,
2847 * by mem_cgroup_kmem_get_cache(), so it is too late to change
2848 * our minds.
2849 *
2850 * This condition will only trigger if the task entered
2851 * memcg_charge_kmem in a sane state, but was OOM-killed during
2852 * __mem_cgroup_try_charge() above. Tasks that were already
2853 * dying when the allocation triggers should have been already
2854 * directed to the root cgroup in memcontrol.h
2855 */
2856 res_counter_charge_nofail(&memcg->res, size, &fail_res);
2857 if (do_swap_account)
2858 res_counter_charge_nofail(&memcg->memsw, size,
2859 &fail_res);
2860 ret = 0;
2861 } else if (ret)
2862 res_counter_uncharge(&memcg->kmem, size);
2863
2864 return ret;
2865}
2866
2867static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
2868{
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002869 res_counter_uncharge(&memcg->res, size);
2870 if (do_swap_account)
2871 res_counter_uncharge(&memcg->memsw, size);
Glauber Costa7de37682012-12-18 14:22:07 -08002872
2873 /* Not down to 0 */
2874 if (res_counter_uncharge(&memcg->kmem, size))
2875 return;
2876
2877 if (memcg_kmem_test_and_clear_dead(memcg))
2878 mem_cgroup_put(memcg);
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002879}
2880
Glauber Costa2633d7a2012-12-18 14:22:34 -08002881void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep)
2882{
2883 if (!memcg)
2884 return;
2885
2886 mutex_lock(&memcg->slab_caches_mutex);
2887 list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches);
2888 mutex_unlock(&memcg->slab_caches_mutex);
2889}
2890
2891/*
2892 * helper for acessing a memcg's index. It will be used as an index in the
2893 * child cache array in kmem_cache, and also to derive its name. This function
2894 * will return -1 when this is not a kmem-limited memcg.
2895 */
2896int memcg_cache_id(struct mem_cgroup *memcg)
2897{
2898 return memcg ? memcg->kmemcg_id : -1;
2899}
2900
Glauber Costa55007d82012-12-18 14:22:38 -08002901/*
2902 * This ends up being protected by the set_limit mutex, during normal
2903 * operation, because that is its main call site.
2904 *
2905 * But when we create a new cache, we can call this as well if its parent
2906 * is kmem-limited. That will have to hold set_limit_mutex as well.
2907 */
2908int memcg_update_cache_sizes(struct mem_cgroup *memcg)
2909{
2910 int num, ret;
2911
2912 num = ida_simple_get(&kmem_limited_groups,
2913 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
2914 if (num < 0)
2915 return num;
2916 /*
2917 * After this point, kmem_accounted (that we test atomically in
2918 * the beginning of this conditional), is no longer 0. This
2919 * guarantees only one process will set the following boolean
2920 * to true. We don't need test_and_set because we're protected
2921 * by the set_limit_mutex anyway.
2922 */
2923 memcg_kmem_set_activated(memcg);
2924
2925 ret = memcg_update_all_caches(num+1);
2926 if (ret) {
2927 ida_simple_remove(&kmem_limited_groups, num);
2928 memcg_kmem_clear_activated(memcg);
2929 return ret;
2930 }
2931
2932 memcg->kmemcg_id = num;
2933 INIT_LIST_HEAD(&memcg->memcg_slab_caches);
2934 mutex_init(&memcg->slab_caches_mutex);
2935 return 0;
2936}
2937
2938static size_t memcg_caches_array_size(int num_groups)
2939{
2940 ssize_t size;
2941 if (num_groups <= 0)
2942 return 0;
2943
2944 size = 2 * num_groups;
2945 if (size < MEMCG_CACHES_MIN_SIZE)
2946 size = MEMCG_CACHES_MIN_SIZE;
2947 else if (size > MEMCG_CACHES_MAX_SIZE)
2948 size = MEMCG_CACHES_MAX_SIZE;
2949
2950 return size;
2951}
2952
2953/*
2954 * We should update the current array size iff all caches updates succeed. This
2955 * can only be done from the slab side. The slab mutex needs to be held when
2956 * calling this.
2957 */
2958void memcg_update_array_size(int num)
2959{
2960 if (num > memcg_limited_groups_array_size)
2961 memcg_limited_groups_array_size = memcg_caches_array_size(num);
2962}
2963
2964int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
2965{
2966 struct memcg_cache_params *cur_params = s->memcg_params;
2967
2968 VM_BUG_ON(s->memcg_params && !s->memcg_params->is_root_cache);
2969
2970 if (num_groups > memcg_limited_groups_array_size) {
2971 int i;
2972 ssize_t size = memcg_caches_array_size(num_groups);
2973
2974 size *= sizeof(void *);
2975 size += sizeof(struct memcg_cache_params);
2976
2977 s->memcg_params = kzalloc(size, GFP_KERNEL);
2978 if (!s->memcg_params) {
2979 s->memcg_params = cur_params;
2980 return -ENOMEM;
2981 }
2982
2983 s->memcg_params->is_root_cache = true;
2984
2985 /*
2986 * There is the chance it will be bigger than
2987 * memcg_limited_groups_array_size, if we failed an allocation
2988 * in a cache, in which case all caches updated before it, will
2989 * have a bigger array.
2990 *
2991 * But if that is the case, the data after
2992 * memcg_limited_groups_array_size is certainly unused
2993 */
2994 for (i = 0; i < memcg_limited_groups_array_size; i++) {
2995 if (!cur_params->memcg_caches[i])
2996 continue;
2997 s->memcg_params->memcg_caches[i] =
2998 cur_params->memcg_caches[i];
2999 }
3000
3001 /*
3002 * Ideally, we would wait until all caches succeed, and only
3003 * then free the old one. But this is not worth the extra
3004 * pointer per-cache we'd have to have for this.
3005 *
3006 * It is not a big deal if some caches are left with a size
3007 * bigger than the others. And all updates will reset this
3008 * anyway.
3009 */
3010 kfree(cur_params);
3011 }
3012 return 0;
3013}
3014
Glauber Costa943a4512012-12-18 14:23:03 -08003015int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s,
3016 struct kmem_cache *root_cache)
Glauber Costa2633d7a2012-12-18 14:22:34 -08003017{
3018 size_t size = sizeof(struct memcg_cache_params);
3019
3020 if (!memcg_kmem_enabled())
3021 return 0;
3022
Glauber Costa55007d82012-12-18 14:22:38 -08003023 if (!memcg)
3024 size += memcg_limited_groups_array_size * sizeof(void *);
3025
Glauber Costa2633d7a2012-12-18 14:22:34 -08003026 s->memcg_params = kzalloc(size, GFP_KERNEL);
3027 if (!s->memcg_params)
3028 return -ENOMEM;
3029
Glauber Costa943a4512012-12-18 14:23:03 -08003030 if (memcg) {
Glauber Costa2633d7a2012-12-18 14:22:34 -08003031 s->memcg_params->memcg = memcg;
Glauber Costa943a4512012-12-18 14:23:03 -08003032 s->memcg_params->root_cache = root_cache;
3033 }
Glauber Costa2633d7a2012-12-18 14:22:34 -08003034 return 0;
3035}
3036
3037void memcg_release_cache(struct kmem_cache *s)
3038{
Glauber Costad7f25f82012-12-18 14:22:40 -08003039 struct kmem_cache *root;
3040 struct mem_cgroup *memcg;
3041 int id;
3042
3043 /*
3044 * This happens, for instance, when a root cache goes away before we
3045 * add any memcg.
3046 */
3047 if (!s->memcg_params)
3048 return;
3049
3050 if (s->memcg_params->is_root_cache)
3051 goto out;
3052
3053 memcg = s->memcg_params->memcg;
3054 id = memcg_cache_id(memcg);
3055
3056 root = s->memcg_params->root_cache;
3057 root->memcg_params->memcg_caches[id] = NULL;
3058 mem_cgroup_put(memcg);
3059
3060 mutex_lock(&memcg->slab_caches_mutex);
3061 list_del(&s->memcg_params->list);
3062 mutex_unlock(&memcg->slab_caches_mutex);
3063
3064out:
Glauber Costa2633d7a2012-12-18 14:22:34 -08003065 kfree(s->memcg_params);
3066}
3067
Glauber Costa0e9d92f2012-12-18 14:22:42 -08003068/*
3069 * During the creation a new cache, we need to disable our accounting mechanism
3070 * altogether. This is true even if we are not creating, but rather just
3071 * enqueing new caches to be created.
3072 *
3073 * This is because that process will trigger allocations; some visible, like
3074 * explicit kmallocs to auxiliary data structures, name strings and internal
3075 * cache structures; some well concealed, like INIT_WORK() that can allocate
3076 * objects during debug.
3077 *
3078 * If any allocation happens during memcg_kmem_get_cache, we will recurse back
3079 * to it. This may not be a bounded recursion: since the first cache creation
3080 * failed to complete (waiting on the allocation), we'll just try to create the
3081 * cache again, failing at the same point.
3082 *
3083 * memcg_kmem_get_cache is prepared to abort after seeing a positive count of
3084 * memcg_kmem_skip_account. So we enclose anything that might allocate memory
3085 * inside the following two functions.
3086 */
3087static inline void memcg_stop_kmem_account(void)
3088{
3089 VM_BUG_ON(!current->mm);
3090 current->memcg_kmem_skip_account++;
3091}
3092
3093static inline void memcg_resume_kmem_account(void)
3094{
3095 VM_BUG_ON(!current->mm);
3096 current->memcg_kmem_skip_account--;
3097}
3098
Glauber Costa1f458cb2012-12-18 14:22:50 -08003099static void kmem_cache_destroy_work_func(struct work_struct *w)
3100{
3101 struct kmem_cache *cachep;
3102 struct memcg_cache_params *p;
3103
3104 p = container_of(w, struct memcg_cache_params, destroy);
3105
3106 cachep = memcg_params_to_cache(p);
3107
Glauber Costa22933152012-12-18 14:22:59 -08003108 /*
3109 * If we get down to 0 after shrink, we could delete right away.
3110 * However, memcg_release_pages() already puts us back in the workqueue
3111 * in that case. If we proceed deleting, we'll get a dangling
3112 * reference, and removing the object from the workqueue in that case
3113 * is unnecessary complication. We are not a fast path.
3114 *
3115 * Note that this case is fundamentally different from racing with
3116 * shrink_slab(): if memcg_cgroup_destroy_cache() is called in
3117 * kmem_cache_shrink, not only we would be reinserting a dead cache
3118 * into the queue, but doing so from inside the worker racing to
3119 * destroy it.
3120 *
3121 * So if we aren't down to zero, we'll just schedule a worker and try
3122 * again
3123 */
3124 if (atomic_read(&cachep->memcg_params->nr_pages) != 0) {
3125 kmem_cache_shrink(cachep);
3126 if (atomic_read(&cachep->memcg_params->nr_pages) == 0)
3127 return;
3128 } else
Glauber Costa1f458cb2012-12-18 14:22:50 -08003129 kmem_cache_destroy(cachep);
3130}
3131
3132void mem_cgroup_destroy_cache(struct kmem_cache *cachep)
3133{
3134 if (!cachep->memcg_params->dead)
3135 return;
3136
3137 /*
Glauber Costa22933152012-12-18 14:22:59 -08003138 * There are many ways in which we can get here.
3139 *
3140 * We can get to a memory-pressure situation while the delayed work is
3141 * still pending to run. The vmscan shrinkers can then release all
3142 * cache memory and get us to destruction. If this is the case, we'll
3143 * be executed twice, which is a bug (the second time will execute over
3144 * bogus data). In this case, cancelling the work should be fine.
3145 *
3146 * But we can also get here from the worker itself, if
3147 * kmem_cache_shrink is enough to shake all the remaining objects and
3148 * get the page count to 0. In this case, we'll deadlock if we try to
3149 * cancel the work (the worker runs with an internal lock held, which
3150 * is the same lock we would hold for cancel_work_sync().)
3151 *
3152 * Since we can't possibly know who got us here, just refrain from
3153 * running if there is already work pending
3154 */
3155 if (work_pending(&cachep->memcg_params->destroy))
3156 return;
3157 /*
Glauber Costa1f458cb2012-12-18 14:22:50 -08003158 * We have to defer the actual destroying to a workqueue, because
3159 * we might currently be in a context that cannot sleep.
3160 */
3161 schedule_work(&cachep->memcg_params->destroy);
3162}
3163
Glauber Costad7f25f82012-12-18 14:22:40 -08003164static char *memcg_cache_name(struct mem_cgroup *memcg, struct kmem_cache *s)
3165{
3166 char *name;
3167 struct dentry *dentry;
3168
3169 rcu_read_lock();
3170 dentry = rcu_dereference(memcg->css.cgroup->dentry);
3171 rcu_read_unlock();
3172
3173 BUG_ON(dentry == NULL);
3174
3175 name = kasprintf(GFP_KERNEL, "%s(%d:%s)", s->name,
3176 memcg_cache_id(memcg), dentry->d_name.name);
3177
3178 return name;
3179}
3180
3181static struct kmem_cache *kmem_cache_dup(struct mem_cgroup *memcg,
3182 struct kmem_cache *s)
3183{
3184 char *name;
3185 struct kmem_cache *new;
3186
3187 name = memcg_cache_name(memcg, s);
3188 if (!name)
3189 return NULL;
3190
3191 new = kmem_cache_create_memcg(memcg, name, s->object_size, s->align,
Glauber Costa943a4512012-12-18 14:23:03 -08003192 (s->flags & ~SLAB_PANIC), s->ctor, s);
Glauber Costad7f25f82012-12-18 14:22:40 -08003193
Glauber Costad79923f2012-12-18 14:22:48 -08003194 if (new)
3195 new->allocflags |= __GFP_KMEMCG;
3196
Glauber Costad7f25f82012-12-18 14:22:40 -08003197 kfree(name);
3198 return new;
3199}
3200
3201/*
3202 * This lock protects updaters, not readers. We want readers to be as fast as
3203 * they can, and they will either see NULL or a valid cache value. Our model
3204 * allow them to see NULL, in which case the root memcg will be selected.
3205 *
3206 * We need this lock because multiple allocations to the same cache from a non
3207 * will span more than one worker. Only one of them can create the cache.
3208 */
3209static DEFINE_MUTEX(memcg_cache_mutex);
3210static struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *memcg,
3211 struct kmem_cache *cachep)
3212{
3213 struct kmem_cache *new_cachep;
3214 int idx;
3215
3216 BUG_ON(!memcg_can_account_kmem(memcg));
3217
3218 idx = memcg_cache_id(memcg);
3219
3220 mutex_lock(&memcg_cache_mutex);
3221 new_cachep = cachep->memcg_params->memcg_caches[idx];
3222 if (new_cachep)
3223 goto out;
3224
3225 new_cachep = kmem_cache_dup(memcg, cachep);
Glauber Costad7f25f82012-12-18 14:22:40 -08003226 if (new_cachep == NULL) {
3227 new_cachep = cachep;
3228 goto out;
3229 }
3230
3231 mem_cgroup_get(memcg);
Glauber Costa1f458cb2012-12-18 14:22:50 -08003232 atomic_set(&new_cachep->memcg_params->nr_pages , 0);
Glauber Costad7f25f82012-12-18 14:22:40 -08003233
3234 cachep->memcg_params->memcg_caches[idx] = new_cachep;
3235 /*
3236 * the readers won't lock, make sure everybody sees the updated value,
3237 * so they won't put stuff in the queue again for no reason
3238 */
3239 wmb();
3240out:
3241 mutex_unlock(&memcg_cache_mutex);
3242 return new_cachep;
3243}
3244
Glauber Costa7cf27982012-12-18 14:22:55 -08003245void kmem_cache_destroy_memcg_children(struct kmem_cache *s)
3246{
3247 struct kmem_cache *c;
3248 int i;
3249
3250 if (!s->memcg_params)
3251 return;
3252 if (!s->memcg_params->is_root_cache)
3253 return;
3254
3255 /*
3256 * If the cache is being destroyed, we trust that there is no one else
3257 * requesting objects from it. Even if there are, the sanity checks in
3258 * kmem_cache_destroy should caught this ill-case.
3259 *
3260 * Still, we don't want anyone else freeing memcg_caches under our
3261 * noses, which can happen if a new memcg comes to life. As usual,
3262 * we'll take the set_limit_mutex to protect ourselves against this.
3263 */
3264 mutex_lock(&set_limit_mutex);
3265 for (i = 0; i < memcg_limited_groups_array_size; i++) {
3266 c = s->memcg_params->memcg_caches[i];
3267 if (!c)
3268 continue;
3269
3270 /*
3271 * We will now manually delete the caches, so to avoid races
3272 * we need to cancel all pending destruction workers and
3273 * proceed with destruction ourselves.
3274 *
3275 * kmem_cache_destroy() will call kmem_cache_shrink internally,
3276 * and that could spawn the workers again: it is likely that
3277 * the cache still have active pages until this very moment.
3278 * This would lead us back to mem_cgroup_destroy_cache.
3279 *
3280 * But that will not execute at all if the "dead" flag is not
3281 * set, so flip it down to guarantee we are in control.
3282 */
3283 c->memcg_params->dead = false;
Glauber Costa22933152012-12-18 14:22:59 -08003284 cancel_work_sync(&c->memcg_params->destroy);
Glauber Costa7cf27982012-12-18 14:22:55 -08003285 kmem_cache_destroy(c);
3286 }
3287 mutex_unlock(&set_limit_mutex);
3288}
3289
Glauber Costad7f25f82012-12-18 14:22:40 -08003290struct create_work {
3291 struct mem_cgroup *memcg;
3292 struct kmem_cache *cachep;
3293 struct work_struct work;
3294};
3295
Glauber Costa1f458cb2012-12-18 14:22:50 -08003296static void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
3297{
3298 struct kmem_cache *cachep;
3299 struct memcg_cache_params *params;
3300
3301 if (!memcg_kmem_is_active(memcg))
3302 return;
3303
3304 mutex_lock(&memcg->slab_caches_mutex);
3305 list_for_each_entry(params, &memcg->memcg_slab_caches, list) {
3306 cachep = memcg_params_to_cache(params);
3307 cachep->memcg_params->dead = true;
3308 INIT_WORK(&cachep->memcg_params->destroy,
Glauber Costa22933152012-12-18 14:22:59 -08003309 kmem_cache_destroy_work_func);
Glauber Costa1f458cb2012-12-18 14:22:50 -08003310 schedule_work(&cachep->memcg_params->destroy);
3311 }
3312 mutex_unlock(&memcg->slab_caches_mutex);
3313}
3314
Glauber Costad7f25f82012-12-18 14:22:40 -08003315static void memcg_create_cache_work_func(struct work_struct *w)
3316{
3317 struct create_work *cw;
3318
3319 cw = container_of(w, struct create_work, work);
3320 memcg_create_kmem_cache(cw->memcg, cw->cachep);
3321 /* Drop the reference gotten when we enqueued. */
3322 css_put(&cw->memcg->css);
3323 kfree(cw);
3324}
3325
3326/*
3327 * Enqueue the creation of a per-memcg kmem_cache.
3328 * Called with rcu_read_lock.
3329 */
Glauber Costa0e9d92f2012-12-18 14:22:42 -08003330static void __memcg_create_cache_enqueue(struct mem_cgroup *memcg,
3331 struct kmem_cache *cachep)
Glauber Costad7f25f82012-12-18 14:22:40 -08003332{
3333 struct create_work *cw;
3334
3335 cw = kmalloc(sizeof(struct create_work), GFP_NOWAIT);
3336 if (cw == NULL)
3337 return;
3338
3339 /* The corresponding put will be done in the workqueue. */
3340 if (!css_tryget(&memcg->css)) {
3341 kfree(cw);
3342 return;
3343 }
3344
3345 cw->memcg = memcg;
3346 cw->cachep = cachep;
3347
3348 INIT_WORK(&cw->work, memcg_create_cache_work_func);
3349 schedule_work(&cw->work);
3350}
3351
Glauber Costa0e9d92f2012-12-18 14:22:42 -08003352static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
3353 struct kmem_cache *cachep)
3354{
3355 /*
3356 * We need to stop accounting when we kmalloc, because if the
3357 * corresponding kmalloc cache is not yet created, the first allocation
3358 * in __memcg_create_cache_enqueue will recurse.
3359 *
3360 * However, it is better to enclose the whole function. Depending on
3361 * the debugging options enabled, INIT_WORK(), for instance, can
3362 * trigger an allocation. This too, will make us recurse. Because at
3363 * this point we can't allow ourselves back into memcg_kmem_get_cache,
3364 * the safest choice is to do it like this, wrapping the whole function.
3365 */
3366 memcg_stop_kmem_account();
3367 __memcg_create_cache_enqueue(memcg, cachep);
3368 memcg_resume_kmem_account();
3369}
Glauber Costad7f25f82012-12-18 14:22:40 -08003370/*
3371 * Return the kmem_cache we're supposed to use for a slab allocation.
3372 * We try to use the current memcg's version of the cache.
3373 *
3374 * If the cache does not exist yet, if we are the first user of it,
3375 * we either create it immediately, if possible, or create it asynchronously
3376 * in a workqueue.
3377 * In the latter case, we will let the current allocation go through with
3378 * the original cache.
3379 *
3380 * Can't be called in interrupt context or from kernel threads.
3381 * This function needs to be called with rcu_read_lock() held.
3382 */
3383struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep,
3384 gfp_t gfp)
3385{
3386 struct mem_cgroup *memcg;
3387 int idx;
3388
3389 VM_BUG_ON(!cachep->memcg_params);
3390 VM_BUG_ON(!cachep->memcg_params->is_root_cache);
3391
Glauber Costa0e9d92f2012-12-18 14:22:42 -08003392 if (!current->mm || current->memcg_kmem_skip_account)
3393 return cachep;
3394
Glauber Costad7f25f82012-12-18 14:22:40 -08003395 rcu_read_lock();
3396 memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner));
3397 rcu_read_unlock();
3398
3399 if (!memcg_can_account_kmem(memcg))
3400 return cachep;
3401
3402 idx = memcg_cache_id(memcg);
3403
3404 /*
3405 * barrier to mare sure we're always seeing the up to date value. The
3406 * code updating memcg_caches will issue a write barrier to match this.
3407 */
3408 read_barrier_depends();
3409 if (unlikely(cachep->memcg_params->memcg_caches[idx] == NULL)) {
3410 /*
3411 * If we are in a safe context (can wait, and not in interrupt
3412 * context), we could be be predictable and return right away.
3413 * This would guarantee that the allocation being performed
3414 * already belongs in the new cache.
3415 *
3416 * However, there are some clashes that can arrive from locking.
3417 * For instance, because we acquire the slab_mutex while doing
3418 * kmem_cache_dup, this means no further allocation could happen
3419 * with the slab_mutex held.
3420 *
3421 * Also, because cache creation issue get_online_cpus(), this
3422 * creates a lock chain: memcg_slab_mutex -> cpu_hotplug_mutex,
3423 * that ends up reversed during cpu hotplug. (cpuset allocates
3424 * a bunch of GFP_KERNEL memory during cpuup). Due to all that,
3425 * better to defer everything.
3426 */
3427 memcg_create_cache_enqueue(memcg, cachep);
3428 return cachep;
3429 }
3430
3431 return cachep->memcg_params->memcg_caches[idx];
3432}
3433EXPORT_SYMBOL(__memcg_kmem_get_cache);
3434
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003435/*
3436 * We need to verify if the allocation against current->mm->owner's memcg is
3437 * possible for the given order. But the page is not allocated yet, so we'll
3438 * need a further commit step to do the final arrangements.
3439 *
3440 * It is possible for the task to switch cgroups in this mean time, so at
3441 * commit time, we can't rely on task conversion any longer. We'll then use
3442 * the handle argument to return to the caller which cgroup we should commit
3443 * against. We could also return the memcg directly and avoid the pointer
3444 * passing, but a boolean return value gives better semantics considering
3445 * the compiled-out case as well.
3446 *
3447 * Returning true means the allocation is possible.
3448 */
3449bool
3450__memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
3451{
3452 struct mem_cgroup *memcg;
3453 int ret;
3454
3455 *_memcg = NULL;
3456 memcg = try_get_mem_cgroup_from_mm(current->mm);
3457
3458 /*
3459 * very rare case described in mem_cgroup_from_task. Unfortunately there
3460 * isn't much we can do without complicating this too much, and it would
3461 * be gfp-dependent anyway. Just let it go
3462 */
3463 if (unlikely(!memcg))
3464 return true;
3465
3466 if (!memcg_can_account_kmem(memcg)) {
3467 css_put(&memcg->css);
3468 return true;
3469 }
3470
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003471 ret = memcg_charge_kmem(memcg, gfp, PAGE_SIZE << order);
3472 if (!ret)
3473 *_memcg = memcg;
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003474
3475 css_put(&memcg->css);
3476 return (ret == 0);
3477}
3478
3479void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg,
3480 int order)
3481{
3482 struct page_cgroup *pc;
3483
3484 VM_BUG_ON(mem_cgroup_is_root(memcg));
3485
3486 /* The page allocation failed. Revert */
3487 if (!page) {
3488 memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003489 return;
3490 }
3491
3492 pc = lookup_page_cgroup(page);
3493 lock_page_cgroup(pc);
3494 pc->mem_cgroup = memcg;
3495 SetPageCgroupUsed(pc);
3496 unlock_page_cgroup(pc);
3497}
3498
3499void __memcg_kmem_uncharge_pages(struct page *page, int order)
3500{
3501 struct mem_cgroup *memcg = NULL;
3502 struct page_cgroup *pc;
3503
3504
3505 pc = lookup_page_cgroup(page);
3506 /*
3507 * Fast unlocked return. Theoretically might have changed, have to
3508 * check again after locking.
3509 */
3510 if (!PageCgroupUsed(pc))
3511 return;
3512
3513 lock_page_cgroup(pc);
3514 if (PageCgroupUsed(pc)) {
3515 memcg = pc->mem_cgroup;
3516 ClearPageCgroupUsed(pc);
3517 }
3518 unlock_page_cgroup(pc);
3519
3520 /*
3521 * We trust that only if there is a memcg associated with the page, it
3522 * is a valid allocation
3523 */
3524 if (!memcg)
3525 return;
3526
3527 VM_BUG_ON(mem_cgroup_is_root(memcg));
3528 memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003529}
Glauber Costa1f458cb2012-12-18 14:22:50 -08003530#else
3531static inline void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
3532{
3533}
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003534#endif /* CONFIG_MEMCG_KMEM */
3535
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003536#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3537
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -07003538#define PCGF_NOCOPY_AT_SPLIT (1 << PCG_LOCK | 1 << PCG_MIGRATION)
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003539/*
3540 * Because tail pages are not marked as "used", set it. We're under
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003541 * zone->lru_lock, 'splitting on pmd' and compound_lock.
3542 * charge/uncharge will be never happen and move_account() is done under
3543 * compound_lock(), so we don't have to take care of races.
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003544 */
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003545void mem_cgroup_split_huge_fixup(struct page *head)
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003546{
3547 struct page_cgroup *head_pc = lookup_page_cgroup(head);
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003548 struct page_cgroup *pc;
3549 int i;
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003550
KAMEZAWA Hiroyuki3d37c4a2011-01-25 15:07:28 -08003551 if (mem_cgroup_disabled())
3552 return;
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003553 for (i = 1; i < HPAGE_PMD_NR; i++) {
3554 pc = head_pc + i;
3555 pc->mem_cgroup = head_pc->mem_cgroup;
3556 smp_wmb();/* see __commit_charge() */
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003557 pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
3558 }
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003559}
Hugh Dickins12d27102012-01-12 17:19:52 -08003560#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003561
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003562/**
Johannes Weinerde3638d2011-03-23 16:42:28 -07003563 * mem_cgroup_move_account - move account of the page
Johannes Weiner5564e882011-03-23 16:42:29 -07003564 * @page: the page
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003565 * @nr_pages: number of regular pages (>1 for huge pages)
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003566 * @pc: page_cgroup of the page.
3567 * @from: mem_cgroup which the page is moved from.
3568 * @to: mem_cgroup which the page is moved to. @from != @to.
3569 *
3570 * The caller must confirm following.
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003571 * - page is not on LRU (isolate_page() is useful.)
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003572 * - compound_lock is held when nr_pages > 1
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003573 *
KAMEZAWA Hiroyuki2f3479b2012-05-29 15:07:04 -07003574 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
3575 * from old cgroup.
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003576 */
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003577static int mem_cgroup_move_account(struct page *page,
3578 unsigned int nr_pages,
3579 struct page_cgroup *pc,
3580 struct mem_cgroup *from,
KAMEZAWA Hiroyuki2f3479b2012-05-29 15:07:04 -07003581 struct mem_cgroup *to)
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003582{
Johannes Weinerde3638d2011-03-23 16:42:28 -07003583 unsigned long flags;
3584 int ret;
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07003585 bool anon = PageAnon(page);
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08003586
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003587 VM_BUG_ON(from == to);
Johannes Weiner5564e882011-03-23 16:42:29 -07003588 VM_BUG_ON(PageLRU(page));
Johannes Weinerde3638d2011-03-23 16:42:28 -07003589 /*
3590 * The page is isolated from LRU. So, collapse function
3591 * will not handle this page. But page splitting can happen.
3592 * Do this check under compound_page_lock(). The caller should
3593 * hold it.
3594 */
3595 ret = -EBUSY;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003596 if (nr_pages > 1 && !PageTransHuge(page))
Johannes Weinerde3638d2011-03-23 16:42:28 -07003597 goto out;
3598
3599 lock_page_cgroup(pc);
3600
3601 ret = -EINVAL;
3602 if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
3603 goto unlock;
3604
KAMEZAWA Hiroyuki312734c02012-03-21 16:34:24 -07003605 move_lock_mem_cgroup(from, &flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003606
KAMEZAWA Hiroyuki2ff76f12012-03-21 16:34:25 -07003607 if (!anon && page_mapped(page)) {
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08003608 /* Update mapped_file data for mem_cgroup */
3609 preempt_disable();
3610 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
3611 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
3612 preempt_enable();
Balbir Singhd69b0422009-06-17 16:26:34 -07003613 }
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07003614 mem_cgroup_charge_statistics(from, anon, -nr_pages);
Balbir Singhd69b0422009-06-17 16:26:34 -07003615
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08003616 /* caller should have done css_get */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003617 pc->mem_cgroup = to;
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07003618 mem_cgroup_charge_statistics(to, anon, nr_pages);
KAMEZAWA Hiroyuki312734c02012-03-21 16:34:24 -07003619 move_unlock_mem_cgroup(from, &flags);
Johannes Weinerde3638d2011-03-23 16:42:28 -07003620 ret = 0;
3621unlock:
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08003622 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08003623 /*
3624 * check events
3625 */
Johannes Weiner5564e882011-03-23 16:42:29 -07003626 memcg_check_events(to, page);
3627 memcg_check_events(from, page);
Johannes Weinerde3638d2011-03-23 16:42:28 -07003628out:
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003629 return ret;
3630}
3631
Michal Hocko2ef37d32012-10-26 13:37:30 +02003632/**
3633 * mem_cgroup_move_parent - moves page to the parent group
3634 * @page: the page to move
3635 * @pc: page_cgroup of the page
3636 * @child: page's cgroup
3637 *
3638 * move charges to its parent or the root cgroup if the group has no
3639 * parent (aka use_hierarchy==0).
3640 * Although this might fail (get_page_unless_zero, isolate_lru_page or
3641 * mem_cgroup_move_account fails) the failure is always temporary and
3642 * it signals a race with a page removal/uncharge or migration. In the
3643 * first case the page is on the way out and it will vanish from the LRU
3644 * on the next attempt and the call should be retried later.
3645 * Isolation from the LRU fails only if page has been isolated from
3646 * the LRU since we looked at it and that usually means either global
3647 * reclaim or migration going on. The page will either get back to the
3648 * LRU or vanish.
3649 * Finaly mem_cgroup_move_account fails only if the page got uncharged
3650 * (!PageCgroupUsed) or moved to a different group. The page will
3651 * disappear in the next attempt.
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003652 */
Johannes Weiner5564e882011-03-23 16:42:29 -07003653static int mem_cgroup_move_parent(struct page *page,
3654 struct page_cgroup *pc,
KAMEZAWA Hiroyuki6068bf02012-07-31 16:42:45 -07003655 struct mem_cgroup *child)
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003656{
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003657 struct mem_cgroup *parent;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003658 unsigned int nr_pages;
Andrew Morton4be44892011-03-23 16:42:39 -07003659 unsigned long uninitialized_var(flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003660 int ret;
3661
Michal Hockod8423012012-10-26 13:37:29 +02003662 VM_BUG_ON(mem_cgroup_is_root(child));
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003663
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08003664 ret = -EBUSY;
3665 if (!get_page_unless_zero(page))
3666 goto out;
3667 if (isolate_lru_page(page))
3668 goto put;
KAMEZAWA Hiroyuki52dbb902011-01-25 15:07:29 -08003669
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003670 nr_pages = hpage_nr_pages(page);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003671
KAMEZAWA Hiroyukicc926f72012-05-29 15:07:04 -07003672 parent = parent_mem_cgroup(child);
3673 /*
3674 * If no parent, move charges to root cgroup.
3675 */
3676 if (!parent)
3677 parent = root_mem_cgroup;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003678
Michal Hocko2ef37d32012-10-26 13:37:30 +02003679 if (nr_pages > 1) {
3680 VM_BUG_ON(!PageTransHuge(page));
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08003681 flags = compound_lock_irqsave(page);
Michal Hocko2ef37d32012-10-26 13:37:30 +02003682 }
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08003683
KAMEZAWA Hiroyukicc926f72012-05-29 15:07:04 -07003684 ret = mem_cgroup_move_account(page, nr_pages,
KAMEZAWA Hiroyuki2f3479b2012-05-29 15:07:04 -07003685 pc, child, parent);
KAMEZAWA Hiroyukicc926f72012-05-29 15:07:04 -07003686 if (!ret)
3687 __mem_cgroup_cancel_local_charge(child, nr_pages);
Jesper Juhl8dba4742011-01-25 15:07:24 -08003688
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003689 if (nr_pages > 1)
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08003690 compound_unlock_irqrestore(page, flags);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003691 putback_lru_page(page);
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08003692put:
Daisuke Nishimura40d58132009-01-15 13:51:12 -08003693 put_page(page);
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08003694out:
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003695 return ret;
3696}
3697
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003698/*
3699 * Charge the memory controller for page usage.
3700 * Return
3701 * 0 if the charge was successful
3702 * < 0 if the cgroup is over its limit
3703 */
3704static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07003705 gfp_t gfp_mask, enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003706{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003707 struct mem_cgroup *memcg = NULL;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003708 unsigned int nr_pages = 1;
Johannes Weiner8493ae42011-02-01 15:52:44 -08003709 bool oom = true;
3710 int ret;
Andrea Arcangeliec168512011-01-13 15:46:56 -08003711
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08003712 if (PageTransHuge(page)) {
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003713 nr_pages <<= compound_order(page);
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08003714 VM_BUG_ON(!PageTransHuge(page));
Johannes Weiner8493ae42011-02-01 15:52:44 -08003715 /*
3716 * Never OOM-kill a process for a huge page. The
3717 * fault handler will fall back to regular pages.
3718 */
3719 oom = false;
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08003720 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003721
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003722 ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08003723 if (ret == -ENOMEM)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003724 return ret;
Johannes Weinerce587e62012-04-24 20:22:33 +02003725 __mem_cgroup_commit_charge(memcg, page, nr_pages, ctype, false);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003726 return 0;
3727}
3728
3729int mem_cgroup_newpage_charge(struct page *page,
3730 struct mm_struct *mm, gfp_t gfp_mask)
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08003731{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08003732 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07003733 return 0;
Johannes Weiner7a0524c2012-01-12 17:18:43 -08003734 VM_BUG_ON(page_mapped(page));
3735 VM_BUG_ON(page->mapping && !PageAnon(page));
3736 VM_BUG_ON(!mm);
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08003737 return mem_cgroup_charge_common(page, mm, gfp_mask,
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -07003738 MEM_CGROUP_CHARGE_TYPE_ANON);
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08003739}
3740
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08003741/*
3742 * While swap-in, try_charge -> commit or cancel, the page is locked.
3743 * And when try_charge() successfully returns, one refcnt to memcg without
Uwe Kleine-König21ae2952009-10-07 15:21:09 +02003744 * struct page_cgroup is acquired. This refcnt will be consumed by
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08003745 * "commit()" or removed by "cancel()"
3746 */
Johannes Weiner0435a2f2012-07-31 16:45:43 -07003747static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm,
3748 struct page *page,
3749 gfp_t mask,
3750 struct mem_cgroup **memcgp)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003751{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003752 struct mem_cgroup *memcg;
Johannes Weiner90deb782012-07-31 16:45:47 -07003753 struct page_cgroup *pc;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08003754 int ret;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003755
Johannes Weiner90deb782012-07-31 16:45:47 -07003756 pc = lookup_page_cgroup(page);
3757 /*
3758 * Every swap fault against a single page tries to charge the
3759 * page, bail as early as possible. shmem_unuse() encounters
3760 * already charged pages, too. The USED bit is protected by
3761 * the page lock, which serializes swap cache removal, which
3762 * in turn serializes uncharging.
3763 */
3764 if (PageCgroupUsed(pc))
3765 return 0;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003766 if (!do_swap_account)
3767 goto charge_cur_mm;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003768 memcg = try_get_mem_cgroup_from_page(page);
3769 if (!memcg)
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08003770 goto charge_cur_mm;
Johannes Weiner72835c82012-01-12 17:18:32 -08003771 *memcgp = memcg;
3772 ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003773 css_put(&memcg->css);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08003774 if (ret == -EINTR)
3775 ret = 0;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08003776 return ret;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003777charge_cur_mm:
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08003778 ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
3779 if (ret == -EINTR)
3780 ret = 0;
3781 return ret;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003782}
3783
Johannes Weiner0435a2f2012-07-31 16:45:43 -07003784int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page,
3785 gfp_t gfp_mask, struct mem_cgroup **memcgp)
3786{
3787 *memcgp = NULL;
3788 if (mem_cgroup_disabled())
3789 return 0;
Johannes Weinerbdf4f4d2012-07-31 16:45:50 -07003790 /*
3791 * A racing thread's fault, or swapoff, may have already
3792 * updated the pte, and even removed page from swap cache: in
3793 * those cases unuse_pte()'s pte_same() test will fail; but
3794 * there's also a KSM case which does need to charge the page.
3795 */
3796 if (!PageSwapCache(page)) {
3797 int ret;
3798
3799 ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, memcgp, true);
3800 if (ret == -EINTR)
3801 ret = 0;
3802 return ret;
3803 }
Johannes Weiner0435a2f2012-07-31 16:45:43 -07003804 return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp);
3805}
3806
Johannes Weiner827a03d2012-07-31 16:45:36 -07003807void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
3808{
3809 if (mem_cgroup_disabled())
3810 return;
3811 if (!memcg)
3812 return;
3813 __mem_cgroup_cancel_charge(memcg, 1);
3814}
3815
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07003816static void
Johannes Weiner72835c82012-01-12 17:18:32 -08003817__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07003818 enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003819{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08003820 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003821 return;
Johannes Weiner72835c82012-01-12 17:18:32 -08003822 if (!memcg)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003823 return;
KAMEZAWA Hiroyuki5a6475a2011-03-23 16:42:42 -07003824
Johannes Weinerce587e62012-04-24 20:22:33 +02003825 __mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003826 /*
3827 * Now swap is on-memory. This means this page may be
3828 * counted both as mem and swap....double count.
KAMEZAWA Hiroyuki03f3c432009-01-07 18:08:31 -08003829 * Fix it by uncharging from memsw. Basically, this SwapCache is stable
3830 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
3831 * may call delete_from_swap_cache() before reach here.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003832 */
KAMEZAWA Hiroyuki03f3c432009-01-07 18:08:31 -08003833 if (do_swap_account && PageSwapCache(page)) {
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003834 swp_entry_t ent = {.val = page_private(page)};
Hugh Dickins86493002012-05-29 15:06:52 -07003835 mem_cgroup_uncharge_swap(ent);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003836 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003837}
3838
Johannes Weiner72835c82012-01-12 17:18:32 -08003839void mem_cgroup_commit_charge_swapin(struct page *page,
3840 struct mem_cgroup *memcg)
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07003841{
Johannes Weiner72835c82012-01-12 17:18:32 -08003842 __mem_cgroup_commit_charge_swapin(page, memcg,
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -07003843 MEM_CGROUP_CHARGE_TYPE_ANON);
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07003844}
3845
Johannes Weiner827a03d2012-07-31 16:45:36 -07003846int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
3847 gfp_t gfp_mask)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003848{
Johannes Weiner827a03d2012-07-31 16:45:36 -07003849 struct mem_cgroup *memcg = NULL;
3850 enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
3851 int ret;
3852
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08003853 if (mem_cgroup_disabled())
Johannes Weiner827a03d2012-07-31 16:45:36 -07003854 return 0;
3855 if (PageCompound(page))
3856 return 0;
3857
Johannes Weiner827a03d2012-07-31 16:45:36 -07003858 if (!PageSwapCache(page))
3859 ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
3860 else { /* page is swapcache/shmem */
Johannes Weiner0435a2f2012-07-31 16:45:43 -07003861 ret = __mem_cgroup_try_charge_swapin(mm, page,
3862 gfp_mask, &memcg);
Johannes Weiner827a03d2012-07-31 16:45:36 -07003863 if (!ret)
3864 __mem_cgroup_commit_charge_swapin(page, memcg, type);
3865 }
3866 return ret;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003867}
3868
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003869static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003870 unsigned int nr_pages,
3871 const enum charge_type ctype)
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003872{
3873 struct memcg_batch_info *batch = NULL;
3874 bool uncharge_memsw = true;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003875
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003876 /* If swapout, usage of swap doesn't decrease */
3877 if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
3878 uncharge_memsw = false;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003879
3880 batch = &current->memcg_batch;
3881 /*
3882 * In usual, we do css_get() when we remember memcg pointer.
3883 * But in this case, we keep res->usage until end of a series of
3884 * uncharges. Then, it's ok to ignore memcg's refcnt.
3885 */
3886 if (!batch->memcg)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003887 batch->memcg = memcg;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003888 /*
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003889 * do_batch > 0 when unmapping pages or inode invalidate/truncate.
Lucas De Marchi25985ed2011-03-30 22:57:33 -03003890 * In those cases, all pages freed continuously can be expected to be in
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003891 * the same cgroup and we have chance to coalesce uncharges.
3892 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
3893 * because we want to do uncharge as soon as possible.
3894 */
3895
3896 if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
3897 goto direct_uncharge;
3898
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003899 if (nr_pages > 1)
Andrea Arcangeliec168512011-01-13 15:46:56 -08003900 goto direct_uncharge;
3901
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07003902 /*
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003903 * In typical case, batch->memcg == mem. This means we can
3904 * merge a series of uncharges to an uncharge of res_counter.
3905 * If not, we uncharge res_counter ony by one.
3906 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003907 if (batch->memcg != memcg)
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003908 goto direct_uncharge;
3909 /* remember freed charge and uncharge it later */
Johannes Weiner7ffd4ca2011-03-23 16:42:35 -07003910 batch->nr_pages++;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003911 if (uncharge_memsw)
Johannes Weiner7ffd4ca2011-03-23 16:42:35 -07003912 batch->memsw_nr_pages++;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003913 return;
3914direct_uncharge:
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003915 res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003916 if (uncharge_memsw)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003917 res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
3918 if (unlikely(batch->memcg != memcg))
3919 memcg_oom_recover(memcg);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08003920}
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003921
Balbir Singh8697d332008-02-07 00:13:59 -08003922/*
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07003923 * uncharge if !page_mapped(page)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08003924 */
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003925static struct mem_cgroup *
Johannes Weiner0030f532012-07-31 16:45:25 -07003926__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype,
3927 bool end_migration)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08003928{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003929 struct mem_cgroup *memcg = NULL;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003930 unsigned int nr_pages = 1;
3931 struct page_cgroup *pc;
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07003932 bool anon;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08003933
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08003934 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003935 return NULL;
Balbir Singh40779602008-04-04 14:29:59 -07003936
Johannes Weiner0c59b892012-07-31 16:45:31 -07003937 VM_BUG_ON(PageSwapCache(page));
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003938
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08003939 if (PageTransHuge(page)) {
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003940 nr_pages <<= compound_order(page);
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08003941 VM_BUG_ON(!PageTransHuge(page));
3942 }
Balbir Singh8697d332008-02-07 00:13:59 -08003943 /*
Balbir Singh3c541e12008-02-07 00:14:41 -08003944 * Check if our page_cgroup is valid
Balbir Singh8697d332008-02-07 00:13:59 -08003945 */
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003946 pc = lookup_page_cgroup(page);
Johannes Weinercfa44942012-01-12 17:18:38 -08003947 if (unlikely(!PageCgroupUsed(pc)))
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003948 return NULL;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08003949
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003950 lock_page_cgroup(pc);
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003951
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003952 memcg = pc->mem_cgroup;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003953
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003954 if (!PageCgroupUsed(pc))
3955 goto unlock_out;
3956
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07003957 anon = PageAnon(page);
3958
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003959 switch (ctype) {
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -07003960 case MEM_CGROUP_CHARGE_TYPE_ANON:
KAMEZAWA Hiroyuki2ff76f12012-03-21 16:34:25 -07003961 /*
3962 * Generally PageAnon tells if it's the anon statistics to be
3963 * updated; but sometimes e.g. mem_cgroup_uncharge_page() is
3964 * used before page reached the stage of being marked PageAnon.
3965 */
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07003966 anon = true;
3967 /* fallthrough */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07003968 case MEM_CGROUP_CHARGE_TYPE_DROP:
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07003969 /* See mem_cgroup_prepare_migration() */
Johannes Weiner0030f532012-07-31 16:45:25 -07003970 if (page_mapped(page))
3971 goto unlock_out;
3972 /*
3973 * Pages under migration may not be uncharged. But
3974 * end_migration() /must/ be the one uncharging the
3975 * unused post-migration page and so it has to call
3976 * here with the migration bit still set. See the
3977 * res_counter handling below.
3978 */
3979 if (!end_migration && PageCgroupMigration(pc))
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003980 goto unlock_out;
3981 break;
3982 case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
3983 if (!PageAnon(page)) { /* Shared memory */
3984 if (page->mapping && !page_is_file_cache(page))
3985 goto unlock_out;
3986 } else if (page_mapped(page)) /* Anon */
3987 goto unlock_out;
3988 break;
3989 default:
3990 break;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003991 }
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08003992
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07003993 mem_cgroup_charge_statistics(memcg, anon, -nr_pages);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07003994
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07003995 ClearPageCgroupUsed(pc);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08003996 /*
3997 * pc->mem_cgroup is not cleared here. It will be accessed when it's
3998 * freed from LRU. This is safe because uncharged page is expected not
3999 * to be reused (freed soon). Exception is SwapCache, it's handled by
4000 * special functions.
4001 */
Hugh Dickinsb9c565d2008-03-04 14:29:11 -08004002
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004003 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07004004 /*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004005 * even after unlock, we have memcg->res.usage here and this memcg
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07004006 * will never be freed.
4007 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004008 memcg_check_events(memcg, page);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07004009 if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004010 mem_cgroup_swap_statistics(memcg, true);
4011 mem_cgroup_get(memcg);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07004012 }
Johannes Weiner0030f532012-07-31 16:45:25 -07004013 /*
4014 * Migration does not charge the res_counter for the
4015 * replacement page, so leave it alone when phasing out the
4016 * page that is unused after the migration.
4017 */
4018 if (!end_migration && !mem_cgroup_is_root(memcg))
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004019 mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004020
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004021 return memcg;
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004022
4023unlock_out:
4024 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004025 return NULL;
Balbir Singh3c541e12008-02-07 00:14:41 -08004026}
4027
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004028void mem_cgroup_uncharge_page(struct page *page)
4029{
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004030 /* early check. */
4031 if (page_mapped(page))
4032 return;
Johannes Weiner40f23a22012-01-12 17:18:45 -08004033 VM_BUG_ON(page->mapping && !PageAnon(page));
Johannes Weiner0c59b892012-07-31 16:45:31 -07004034 if (PageSwapCache(page))
4035 return;
Johannes Weiner0030f532012-07-31 16:45:25 -07004036 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_ANON, false);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004037}
4038
4039void mem_cgroup_uncharge_cache_page(struct page *page)
4040{
4041 VM_BUG_ON(page_mapped(page));
KAMEZAWA Hiroyukib7abea92008-10-18 20:28:09 -07004042 VM_BUG_ON(page->mapping);
Johannes Weiner0030f532012-07-31 16:45:25 -07004043 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE, false);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004044}
4045
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004046/*
4047 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
4048 * In that cases, pages are freed continuously and we can expect pages
4049 * are in the same memcg. All these calls itself limits the number of
4050 * pages freed at once, then uncharge_start/end() is called properly.
4051 * This may be called prural(2) times in a context,
4052 */
4053
4054void mem_cgroup_uncharge_start(void)
4055{
4056 current->memcg_batch.do_batch++;
4057 /* We can do nest. */
4058 if (current->memcg_batch.do_batch == 1) {
4059 current->memcg_batch.memcg = NULL;
Johannes Weiner7ffd4ca2011-03-23 16:42:35 -07004060 current->memcg_batch.nr_pages = 0;
4061 current->memcg_batch.memsw_nr_pages = 0;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004062 }
4063}
4064
4065void mem_cgroup_uncharge_end(void)
4066{
4067 struct memcg_batch_info *batch = &current->memcg_batch;
4068
4069 if (!batch->do_batch)
4070 return;
4071
4072 batch->do_batch--;
4073 if (batch->do_batch) /* If stacked, do nothing. */
4074 return;
4075
4076 if (!batch->memcg)
4077 return;
4078 /*
4079 * This "batch->memcg" is valid without any css_get/put etc...
4080 * bacause we hide charges behind us.
4081 */
Johannes Weiner7ffd4ca2011-03-23 16:42:35 -07004082 if (batch->nr_pages)
4083 res_counter_uncharge(&batch->memcg->res,
4084 batch->nr_pages * PAGE_SIZE);
4085 if (batch->memsw_nr_pages)
4086 res_counter_uncharge(&batch->memcg->memsw,
4087 batch->memsw_nr_pages * PAGE_SIZE);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004088 memcg_oom_recover(batch->memcg);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004089 /* forget this pointer (for sanity check) */
4090 batch->memcg = NULL;
4091}
4092
Daisuke Nishimurae767e052009-05-28 14:34:28 -07004093#ifdef CONFIG_SWAP
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004094/*
Daisuke Nishimurae767e052009-05-28 14:34:28 -07004095 * called after __delete_from_swap_cache() and drop "page" account.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004096 * memcg information is recorded to swap_cgroup of "ent"
4097 */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07004098void
4099mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004100{
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004101 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07004102 int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004103
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07004104 if (!swapout) /* this was a swap cache but the swap is unused ! */
4105 ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
4106
Johannes Weiner0030f532012-07-31 16:45:25 -07004107 memcg = __mem_cgroup_uncharge_common(page, ctype, false);
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07004108
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07004109 /*
4110 * record memcg information, if swapout && memcg != NULL,
4111 * mem_cgroup_get() was called in uncharge().
4112 */
4113 if (do_swap_account && swapout && memcg)
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07004114 swap_cgroup_record(ent, css_id(&memcg->css));
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004115}
Daisuke Nishimurae767e052009-05-28 14:34:28 -07004116#endif
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004117
Andrew Mortonc255a452012-07-31 16:43:02 -07004118#ifdef CONFIG_MEMCG_SWAP
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004119/*
4120 * called from swap_entry_free(). remove record in swap_cgroup and
4121 * uncharge "memsw" account.
4122 */
4123void mem_cgroup_uncharge_swap(swp_entry_t ent)
4124{
4125 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07004126 unsigned short id;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004127
4128 if (!do_swap_account)
4129 return;
4130
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07004131 id = swap_cgroup_record(ent, 0);
4132 rcu_read_lock();
4133 memcg = mem_cgroup_lookup(id);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004134 if (memcg) {
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07004135 /*
4136 * We uncharge this because swap is freed.
4137 * This memcg can be obsolete one. We avoid calling css_tryget
4138 */
Balbir Singh0c3e73e2009-09-23 15:56:42 -07004139 if (!mem_cgroup_is_root(memcg))
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -07004140 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07004141 mem_cgroup_swap_statistics(memcg, false);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004142 mem_cgroup_put(memcg);
4143 }
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07004144 rcu_read_unlock();
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004145}
Daisuke Nishimura02491442010-03-10 15:22:17 -08004146
4147/**
4148 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
4149 * @entry: swap entry to be moved
4150 * @from: mem_cgroup which the entry is moved from
4151 * @to: mem_cgroup which the entry is moved to
4152 *
4153 * It succeeds only when the swap_cgroup's record for this entry is the same
4154 * as the mem_cgroup's id of @from.
4155 *
4156 * Returns 0 on success, -EINVAL on failure.
4157 *
4158 * The caller must have charged to @to, IOW, called res_counter_charge() about
4159 * both res and memsw, and called css_get().
4160 */
4161static int mem_cgroup_move_swap_account(swp_entry_t entry,
Hugh Dickinse91cbb42012-05-29 15:06:51 -07004162 struct mem_cgroup *from, struct mem_cgroup *to)
Daisuke Nishimura02491442010-03-10 15:22:17 -08004163{
4164 unsigned short old_id, new_id;
4165
4166 old_id = css_id(&from->css);
4167 new_id = css_id(&to->css);
4168
4169 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08004170 mem_cgroup_swap_statistics(from, false);
Daisuke Nishimura02491442010-03-10 15:22:17 -08004171 mem_cgroup_swap_statistics(to, true);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004172 /*
4173 * This function is only called from task migration context now.
4174 * It postpones res_counter and refcount handling till the end
4175 * of task migration(mem_cgroup_clear_mc()) for performance
4176 * improvement. But we cannot postpone mem_cgroup_get(to)
4177 * because if the process that has been moved to @to does
4178 * swap-in, the refcount of @to might be decreased to 0.
4179 */
Daisuke Nishimura02491442010-03-10 15:22:17 -08004180 mem_cgroup_get(to);
Daisuke Nishimura02491442010-03-10 15:22:17 -08004181 return 0;
4182 }
4183 return -EINVAL;
4184}
4185#else
4186static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
Hugh Dickinse91cbb42012-05-29 15:06:51 -07004187 struct mem_cgroup *from, struct mem_cgroup *to)
Daisuke Nishimura02491442010-03-10 15:22:17 -08004188{
4189 return -EINVAL;
4190}
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004191#endif
4192
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08004193/*
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004194 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
4195 * page belongs to.
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08004196 */
Johannes Weiner0030f532012-07-31 16:45:25 -07004197void mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
4198 struct mem_cgroup **memcgp)
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08004199{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004200 struct mem_cgroup *memcg = NULL;
Mel Gormanb32967f2012-11-19 12:35:47 +00004201 unsigned int nr_pages = 1;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07004202 struct page_cgroup *pc;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004203 enum charge_type ctype;
Hugh Dickins8869b8f2008-03-04 14:29:09 -08004204
Johannes Weiner72835c82012-01-12 17:18:32 -08004205 *memcgp = NULL;
KAMEZAWA Hiroyuki56039ef2011-03-23 16:42:19 -07004206
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08004207 if (mem_cgroup_disabled())
Johannes Weiner0030f532012-07-31 16:45:25 -07004208 return;
Balbir Singh40779602008-04-04 14:29:59 -07004209
Mel Gormanb32967f2012-11-19 12:35:47 +00004210 if (PageTransHuge(page))
4211 nr_pages <<= compound_order(page);
4212
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004213 pc = lookup_page_cgroup(page);
4214 lock_page_cgroup(pc);
4215 if (PageCgroupUsed(pc)) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004216 memcg = pc->mem_cgroup;
4217 css_get(&memcg->css);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004218 /*
4219 * At migrating an anonymous page, its mapcount goes down
4220 * to 0 and uncharge() will be called. But, even if it's fully
4221 * unmapped, migration may fail and this page has to be
4222 * charged again. We set MIGRATION flag here and delay uncharge
4223 * until end_migration() is called
4224 *
4225 * Corner Case Thinking
4226 * A)
4227 * When the old page was mapped as Anon and it's unmap-and-freed
4228 * while migration was ongoing.
4229 * If unmap finds the old page, uncharge() of it will be delayed
4230 * until end_migration(). If unmap finds a new page, it's
4231 * uncharged when it make mapcount to be 1->0. If unmap code
4232 * finds swap_migration_entry, the new page will not be mapped
4233 * and end_migration() will find it(mapcount==0).
4234 *
4235 * B)
4236 * When the old page was mapped but migraion fails, the kernel
4237 * remaps it. A charge for it is kept by MIGRATION flag even
4238 * if mapcount goes down to 0. We can do remap successfully
4239 * without charging it again.
4240 *
4241 * C)
4242 * The "old" page is under lock_page() until the end of
4243 * migration, so, the old page itself will not be swapped-out.
4244 * If the new page is swapped out before end_migraton, our
4245 * hook to usual swap-out path will catch the event.
4246 */
4247 if (PageAnon(page))
4248 SetPageCgroupMigration(pc);
Hugh Dickinsb9c565d2008-03-04 14:29:11 -08004249 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004250 unlock_page_cgroup(pc);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004251 /*
4252 * If the page is not charged at this point,
4253 * we return here.
4254 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004255 if (!memcg)
Johannes Weiner0030f532012-07-31 16:45:25 -07004256 return;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004257
Johannes Weiner72835c82012-01-12 17:18:32 -08004258 *memcgp = memcg;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004259 /*
4260 * We charge new page before it's used/mapped. So, even if unlock_page()
4261 * is called before end_migration, we can catch all events on this new
4262 * page. In the case new page is migrated but not remapped, new page's
4263 * mapcount will be finally 0 and we call uncharge in end_migration().
4264 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004265 if (PageAnon(page))
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -07004266 ctype = MEM_CGROUP_CHARGE_TYPE_ANON;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004267 else
Johannes Weiner62ba7442012-07-31 16:45:39 -07004268 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
Johannes Weiner0030f532012-07-31 16:45:25 -07004269 /*
4270 * The page is committed to the memcg, but it's not actually
4271 * charged to the res_counter since we plan on replacing the
4272 * old one and only one page is going to be left afterwards.
4273 */
Mel Gormanb32967f2012-11-19 12:35:47 +00004274 __mem_cgroup_commit_charge(memcg, newpage, nr_pages, ctype, false);
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07004275}
Hugh Dickinsfb59e9f2008-03-04 14:29:16 -08004276
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004277/* remove redundant charge if migration failed*/
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004278void mem_cgroup_end_migration(struct mem_cgroup *memcg,
Daisuke Nishimura50de1dd2011-01-13 15:47:43 -08004279 struct page *oldpage, struct page *newpage, bool migration_ok)
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07004280{
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004281 struct page *used, *unused;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004282 struct page_cgroup *pc;
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07004283 bool anon;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004284
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004285 if (!memcg)
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004286 return;
Tejun Heob25ed602012-11-05 09:16:59 -08004287
Daisuke Nishimura50de1dd2011-01-13 15:47:43 -08004288 if (!migration_ok) {
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004289 used = oldpage;
4290 unused = newpage;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004291 } else {
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004292 used = newpage;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004293 unused = oldpage;
4294 }
Johannes Weiner0030f532012-07-31 16:45:25 -07004295 anon = PageAnon(used);
Johannes Weiner7d188952012-07-31 16:45:34 -07004296 __mem_cgroup_uncharge_common(unused,
4297 anon ? MEM_CGROUP_CHARGE_TYPE_ANON
4298 : MEM_CGROUP_CHARGE_TYPE_CACHE,
4299 true);
Johannes Weiner0030f532012-07-31 16:45:25 -07004300 css_put(&memcg->css);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004301 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004302 * We disallowed uncharge of pages under migration because mapcount
4303 * of the page goes down to zero, temporarly.
4304 * Clear the flag and check the page should be charged.
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004305 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004306 pc = lookup_page_cgroup(oldpage);
4307 lock_page_cgroup(pc);
4308 ClearPageCgroupMigration(pc);
4309 unlock_page_cgroup(pc);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004310
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004311 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004312 * If a page is a file cache, radix-tree replacement is very atomic
4313 * and we can skip this check. When it was an Anon page, its mapcount
4314 * goes down to 0. But because we added MIGRATION flage, it's not
4315 * uncharged yet. There are several case but page->mapcount check
4316 * and USED bit check in mem_cgroup_uncharge_page() will do enough
4317 * check. (see prepare_charge() also)
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004318 */
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07004319 if (anon)
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004320 mem_cgroup_uncharge_page(used);
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08004321}
Pavel Emelianov78fb7462008-02-07 00:13:51 -08004322
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004323/*
4324 * At replace page cache, newpage is not under any memcg but it's on
4325 * LRU. So, this function doesn't touch res_counter but handles LRU
4326 * in correct way. Both pages are locked so we cannot race with uncharge.
4327 */
4328void mem_cgroup_replace_page_cache(struct page *oldpage,
4329 struct page *newpage)
4330{
Hugh Dickinsbde05d12012-05-29 15:06:38 -07004331 struct mem_cgroup *memcg = NULL;
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004332 struct page_cgroup *pc;
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004333 enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004334
4335 if (mem_cgroup_disabled())
4336 return;
4337
4338 pc = lookup_page_cgroup(oldpage);
4339 /* fix accounting on old pages */
4340 lock_page_cgroup(pc);
Hugh Dickinsbde05d12012-05-29 15:06:38 -07004341 if (PageCgroupUsed(pc)) {
4342 memcg = pc->mem_cgroup;
4343 mem_cgroup_charge_statistics(memcg, false, -1);
4344 ClearPageCgroupUsed(pc);
4345 }
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004346 unlock_page_cgroup(pc);
4347
Hugh Dickinsbde05d12012-05-29 15:06:38 -07004348 /*
4349 * When called from shmem_replace_page(), in some cases the
4350 * oldpage has already been charged, and in some cases not.
4351 */
4352 if (!memcg)
4353 return;
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004354 /*
4355 * Even if newpage->mapping was NULL before starting replacement,
4356 * the newpage may be on LRU(or pagevec for LRU) already. We lock
4357 * LRU while we overwrite pc->mem_cgroup.
4358 */
Johannes Weinerce587e62012-04-24 20:22:33 +02004359 __mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004360}
4361
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -07004362#ifdef CONFIG_DEBUG_VM
4363static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
4364{
4365 struct page_cgroup *pc;
4366
4367 pc = lookup_page_cgroup(page);
Johannes Weinercfa44942012-01-12 17:18:38 -08004368 /*
4369 * Can be NULL while feeding pages into the page allocator for
4370 * the first time, i.e. during boot or memory hotplug;
4371 * or when mem_cgroup_disabled().
4372 */
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -07004373 if (likely(pc) && PageCgroupUsed(pc))
4374 return pc;
4375 return NULL;
4376}
4377
4378bool mem_cgroup_bad_page_check(struct page *page)
4379{
4380 if (mem_cgroup_disabled())
4381 return false;
4382
4383 return lookup_page_cgroup_used(page) != NULL;
4384}
4385
4386void mem_cgroup_print_bad_page(struct page *page)
4387{
4388 struct page_cgroup *pc;
4389
4390 pc = lookup_page_cgroup_used(page);
4391 if (pc) {
Hugh Dickins90b3fea2012-01-12 17:19:54 -08004392 printk(KERN_ALERT "pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -07004393 pc, pc->flags, pc->mem_cgroup);
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -07004394 }
4395}
4396#endif
4397
KOSAKI Motohirod38d2a72009-01-06 14:39:44 -08004398static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004399 unsigned long long val)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004400{
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004401 int retry_count;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004402 u64 memswlimit, memlimit;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004403 int ret = 0;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004404 int children = mem_cgroup_count_children(memcg);
4405 u64 curusage, oldusage;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004406 int enlarge;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004407
4408 /*
4409 * For keeping hierarchical_reclaim simple, how long we should retry
4410 * is depends on callers. We set our retry-count to be function
4411 * of # of children which we should visit in this loop.
4412 */
4413 retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
4414
4415 oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004416
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004417 enlarge = 0;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004418 while (retry_count) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004419 if (signal_pending(current)) {
4420 ret = -EINTR;
4421 break;
4422 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004423 /*
4424 * Rather than hide all in some function, I do this in
4425 * open coded manner. You see what this really does.
Wanpeng Liaaad1532012-07-31 16:43:23 -07004426 * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004427 */
4428 mutex_lock(&set_limit_mutex);
4429 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4430 if (memswlimit < val) {
4431 ret = -EINVAL;
4432 mutex_unlock(&set_limit_mutex);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004433 break;
4434 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004435
4436 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4437 if (memlimit < val)
4438 enlarge = 1;
4439
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004440 ret = res_counter_set_limit(&memcg->res, val);
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07004441 if (!ret) {
4442 if (memswlimit == val)
4443 memcg->memsw_is_minimum = true;
4444 else
4445 memcg->memsw_is_minimum = false;
4446 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004447 mutex_unlock(&set_limit_mutex);
4448
4449 if (!ret)
4450 break;
4451
Johannes Weiner56600482012-01-12 17:17:59 -08004452 mem_cgroup_reclaim(memcg, GFP_KERNEL,
4453 MEM_CGROUP_RECLAIM_SHRINK);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004454 curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
4455 /* Usage is reduced ? */
4456 if (curusage >= oldusage)
4457 retry_count--;
4458 else
4459 oldusage = curusage;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004460 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004461 if (!ret && enlarge)
4462 memcg_oom_recover(memcg);
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08004463
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004464 return ret;
4465}
4466
Li Zefan338c8432009-06-17 16:27:15 -07004467static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
4468 unsigned long long val)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004469{
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004470 int retry_count;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004471 u64 memlimit, memswlimit, oldusage, curusage;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004472 int children = mem_cgroup_count_children(memcg);
4473 int ret = -EBUSY;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004474 int enlarge = 0;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004475
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004476 /* see mem_cgroup_resize_res_limit */
4477 retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
4478 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004479 while (retry_count) {
4480 if (signal_pending(current)) {
4481 ret = -EINTR;
4482 break;
4483 }
4484 /*
4485 * Rather than hide all in some function, I do this in
4486 * open coded manner. You see what this really does.
Wanpeng Liaaad1532012-07-31 16:43:23 -07004487 * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004488 */
4489 mutex_lock(&set_limit_mutex);
4490 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4491 if (memlimit > val) {
4492 ret = -EINVAL;
4493 mutex_unlock(&set_limit_mutex);
4494 break;
4495 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004496 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4497 if (memswlimit < val)
4498 enlarge = 1;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004499 ret = res_counter_set_limit(&memcg->memsw, val);
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07004500 if (!ret) {
4501 if (memlimit == val)
4502 memcg->memsw_is_minimum = true;
4503 else
4504 memcg->memsw_is_minimum = false;
4505 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004506 mutex_unlock(&set_limit_mutex);
4507
4508 if (!ret)
4509 break;
4510
Johannes Weiner56600482012-01-12 17:17:59 -08004511 mem_cgroup_reclaim(memcg, GFP_KERNEL,
4512 MEM_CGROUP_RECLAIM_NOSWAP |
4513 MEM_CGROUP_RECLAIM_SHRINK);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004514 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004515 /* Usage is reduced ? */
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004516 if (curusage >= oldusage)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004517 retry_count--;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004518 else
4519 oldusage = curusage;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004520 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004521 if (!ret && enlarge)
4522 memcg_oom_recover(memcg);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004523 return ret;
4524}
4525
Balbir Singh4e416952009-09-23 15:56:39 -07004526unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
Ying Han0ae5e892011-05-26 16:25:25 -07004527 gfp_t gfp_mask,
4528 unsigned long *total_scanned)
Balbir Singh4e416952009-09-23 15:56:39 -07004529{
4530 unsigned long nr_reclaimed = 0;
4531 struct mem_cgroup_per_zone *mz, *next_mz = NULL;
4532 unsigned long reclaimed;
4533 int loop = 0;
4534 struct mem_cgroup_tree_per_zone *mctz;
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07004535 unsigned long long excess;
Ying Han0ae5e892011-05-26 16:25:25 -07004536 unsigned long nr_scanned;
Balbir Singh4e416952009-09-23 15:56:39 -07004537
4538 if (order > 0)
4539 return 0;
4540
KOSAKI Motohiro00918b62010-08-10 18:03:05 -07004541 mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
Balbir Singh4e416952009-09-23 15:56:39 -07004542 /*
4543 * This loop can run a while, specially if mem_cgroup's continuously
4544 * keep exceeding their soft limit and putting the system under
4545 * pressure
4546 */
4547 do {
4548 if (next_mz)
4549 mz = next_mz;
4550 else
4551 mz = mem_cgroup_largest_soft_limit_node(mctz);
4552 if (!mz)
4553 break;
4554
Ying Han0ae5e892011-05-26 16:25:25 -07004555 nr_scanned = 0;
Hugh Dickinsd79154b2012-03-21 16:34:18 -07004556 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
Johannes Weiner56600482012-01-12 17:17:59 -08004557 gfp_mask, &nr_scanned);
Balbir Singh4e416952009-09-23 15:56:39 -07004558 nr_reclaimed += reclaimed;
Ying Han0ae5e892011-05-26 16:25:25 -07004559 *total_scanned += nr_scanned;
Balbir Singh4e416952009-09-23 15:56:39 -07004560 spin_lock(&mctz->lock);
4561
4562 /*
4563 * If we failed to reclaim anything from this memory cgroup
4564 * it is time to move on to the next cgroup
4565 */
4566 next_mz = NULL;
4567 if (!reclaimed) {
4568 do {
4569 /*
4570 * Loop until we find yet another one.
4571 *
4572 * By the time we get the soft_limit lock
4573 * again, someone might have aded the
4574 * group back on the RB tree. Iterate to
4575 * make sure we get a different mem.
4576 * mem_cgroup_largest_soft_limit_node returns
4577 * NULL if no other cgroup is present on
4578 * the tree
4579 */
4580 next_mz =
4581 __mem_cgroup_largest_soft_limit_node(mctz);
Michal Hocko39cc98f2011-05-26 16:25:28 -07004582 if (next_mz == mz)
Hugh Dickinsd79154b2012-03-21 16:34:18 -07004583 css_put(&next_mz->memcg->css);
Michal Hocko39cc98f2011-05-26 16:25:28 -07004584 else /* next_mz == NULL or other memcg */
Balbir Singh4e416952009-09-23 15:56:39 -07004585 break;
4586 } while (1);
4587 }
Hugh Dickinsd79154b2012-03-21 16:34:18 -07004588 __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
4589 excess = res_counter_soft_limit_excess(&mz->memcg->res);
Balbir Singh4e416952009-09-23 15:56:39 -07004590 /*
4591 * One school of thought says that we should not add
4592 * back the node to the tree if reclaim returns 0.
4593 * But our reclaim could return 0, simply because due
4594 * to priority we are exposing a smaller subset of
4595 * memory to reclaim from. Consider this as a longer
4596 * term TODO.
4597 */
KAMEZAWA Hiroyukief8745c2009-10-01 15:44:12 -07004598 /* If excess == 0, no tree ops */
Hugh Dickinsd79154b2012-03-21 16:34:18 -07004599 __mem_cgroup_insert_exceeded(mz->memcg, mz, mctz, excess);
Balbir Singh4e416952009-09-23 15:56:39 -07004600 spin_unlock(&mctz->lock);
Hugh Dickinsd79154b2012-03-21 16:34:18 -07004601 css_put(&mz->memcg->css);
Balbir Singh4e416952009-09-23 15:56:39 -07004602 loop++;
4603 /*
4604 * Could not reclaim anything and there are no more
4605 * mem cgroups to try or we seem to be looping without
4606 * reclaiming anything.
4607 */
4608 if (!nr_reclaimed &&
4609 (next_mz == NULL ||
4610 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
4611 break;
4612 } while (!nr_reclaimed);
4613 if (next_mz)
Hugh Dickinsd79154b2012-03-21 16:34:18 -07004614 css_put(&next_mz->memcg->css);
Balbir Singh4e416952009-09-23 15:56:39 -07004615 return nr_reclaimed;
4616}
4617
Michal Hocko2ef37d32012-10-26 13:37:30 +02004618/**
4619 * mem_cgroup_force_empty_list - clears LRU of a group
4620 * @memcg: group to clear
4621 * @node: NUMA node
4622 * @zid: zone id
4623 * @lru: lru to to clear
4624 *
KAMEZAWA Hiroyuki3c935d12012-07-31 16:42:46 -07004625 * Traverse a specified page_cgroup list and try to drop them all. This doesn't
Michal Hocko2ef37d32012-10-26 13:37:30 +02004626 * reclaim the pages page themselves - pages are moved to the parent (or root)
4627 * group.
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004628 */
Michal Hocko2ef37d32012-10-26 13:37:30 +02004629static void mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004630 int node, int zid, enum lru_list lru)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004631{
Hugh Dickinsbea8c152012-11-16 14:14:54 -08004632 struct lruvec *lruvec;
Michal Hocko2ef37d32012-10-26 13:37:30 +02004633 unsigned long flags;
KAMEZAWA Hiroyuki072c56c12008-02-07 00:14:39 -08004634 struct list_head *list;
Johannes Weiner925b7672012-01-12 17:18:15 -08004635 struct page *busy;
4636 struct zone *zone;
KAMEZAWA Hiroyuki072c56c12008-02-07 00:14:39 -08004637
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004638 zone = &NODE_DATA(node)->node_zones[zid];
Hugh Dickinsbea8c152012-11-16 14:14:54 -08004639 lruvec = mem_cgroup_zone_lruvec(zone, memcg);
4640 list = &lruvec->lists[lru];
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004641
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004642 busy = NULL;
Michal Hocko2ef37d32012-10-26 13:37:30 +02004643 do {
Johannes Weiner925b7672012-01-12 17:18:15 -08004644 struct page_cgroup *pc;
Johannes Weiner5564e882011-03-23 16:42:29 -07004645 struct page *page;
4646
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004647 spin_lock_irqsave(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004648 if (list_empty(list)) {
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004649 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004650 break;
4651 }
Johannes Weiner925b7672012-01-12 17:18:15 -08004652 page = list_entry(list->prev, struct page, lru);
4653 if (busy == page) {
4654 list_move(&page->lru, list);
Thiago Farina648bcc72010-03-05 13:42:04 -08004655 busy = NULL;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004656 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004657 continue;
4658 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004659 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004660
Johannes Weiner925b7672012-01-12 17:18:15 -08004661 pc = lookup_page_cgroup(page);
Johannes Weiner5564e882011-03-23 16:42:29 -07004662
KAMEZAWA Hiroyuki3c935d12012-07-31 16:42:46 -07004663 if (mem_cgroup_move_parent(page, pc, memcg)) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004664 /* found lock contention or "pc" is obsolete. */
Johannes Weiner925b7672012-01-12 17:18:15 -08004665 busy = page;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004666 cond_resched();
4667 } else
4668 busy = NULL;
Michal Hocko2ef37d32012-10-26 13:37:30 +02004669 } while (!list_empty(list));
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004670}
4671
4672/*
Michal Hockoc26251f2012-10-26 13:37:28 +02004673 * make mem_cgroup's charge to be 0 if there is no task by moving
4674 * all the charges and pages to the parent.
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004675 * This enables deleting this mem_cgroup.
Michal Hockoc26251f2012-10-26 13:37:28 +02004676 *
4677 * Caller is responsible for holding css reference on the memcg.
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004678 */
Michal Hockoab5196c2012-10-26 13:37:32 +02004679static void mem_cgroup_reparent_charges(struct mem_cgroup *memcg)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004680{
Michal Hockoc26251f2012-10-26 13:37:28 +02004681 int node, zid;
Glauber Costabea207c2012-12-18 14:22:11 -08004682 u64 usage;
Hugh Dickins8869b8f2008-03-04 14:29:09 -08004683
Daisuke Nishimurafce66472010-01-15 17:01:30 -08004684 do {
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004685 /* This is for making all *used* pages to be on LRU. */
4686 lru_add_drain_all();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004687 drain_all_stock_sync(memcg);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004688 mem_cgroup_start_move(memcg);
Lai Jiangshan31aaea42012-12-12 13:51:27 -08004689 for_each_node_state(node, N_MEMORY) {
Michal Hocko2ef37d32012-10-26 13:37:30 +02004690 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
Hugh Dickinsf156ab92012-03-21 16:34:19 -07004691 enum lru_list lru;
4692 for_each_lru(lru) {
Michal Hocko2ef37d32012-10-26 13:37:30 +02004693 mem_cgroup_force_empty_list(memcg,
Hugh Dickinsf156ab92012-03-21 16:34:19 -07004694 node, zid, lru);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004695 }
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004696 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004697 }
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004698 mem_cgroup_end_move(memcg);
4699 memcg_oom_recover(memcg);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004700 cond_resched();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004701
Michal Hocko2ef37d32012-10-26 13:37:30 +02004702 /*
Glauber Costabea207c2012-12-18 14:22:11 -08004703 * Kernel memory may not necessarily be trackable to a specific
4704 * process. So they are not migrated, and therefore we can't
4705 * expect their value to drop to 0 here.
4706 * Having res filled up with kmem only is enough.
4707 *
Michal Hocko2ef37d32012-10-26 13:37:30 +02004708 * This is a safety check because mem_cgroup_force_empty_list
4709 * could have raced with mem_cgroup_replace_page_cache callers
4710 * so the lru seemed empty but the page could have been added
4711 * right after the check. RES_USAGE should be safe as we always
4712 * charge before adding to the LRU.
4713 */
Glauber Costabea207c2012-12-18 14:22:11 -08004714 usage = res_counter_read_u64(&memcg->res, RES_USAGE) -
4715 res_counter_read_u64(&memcg->kmem, RES_USAGE);
4716 } while (usage > 0);
Michal Hockoc26251f2012-10-26 13:37:28 +02004717}
4718
4719/*
4720 * Reclaims as many pages from the given memcg as possible and moves
4721 * the rest to the parent.
4722 *
4723 * Caller is responsible for holding css reference for memcg.
4724 */
4725static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
4726{
4727 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
4728 struct cgroup *cgrp = memcg->css.cgroup;
4729
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004730 /* returns EBUSY if there is a task or if we come here twice. */
Michal Hockoc26251f2012-10-26 13:37:28 +02004731 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
4732 return -EBUSY;
4733
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004734 /* we call try-to-free pages for make this cgroup empty */
4735 lru_add_drain_all();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004736 /* try to free all pages in this cgroup */
Glauber Costa569530f2012-04-12 12:49:13 -07004737 while (nr_retries && res_counter_read_u64(&memcg->res, RES_USAGE) > 0) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004738 int progress;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004739
Michal Hockoc26251f2012-10-26 13:37:28 +02004740 if (signal_pending(current))
4741 return -EINTR;
4742
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004743 progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
Johannes Weiner185efc02011-09-14 16:21:58 -07004744 false);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004745 if (!progress) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004746 nr_retries--;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004747 /* maybe some writeback is necessary */
Jens Axboe8aa7e842009-07-09 14:52:32 +02004748 congestion_wait(BLK_RW_ASYNC, HZ/10);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004749 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004750
4751 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004752 lru_add_drain();
Michal Hockoab5196c2012-10-26 13:37:32 +02004753 mem_cgroup_reparent_charges(memcg);
4754
4755 return 0;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004756}
4757
Kirill A. Shutemov6bbda352012-05-29 15:06:55 -07004758static int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004759{
Michal Hockoc26251f2012-10-26 13:37:28 +02004760 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4761 int ret;
4762
Michal Hockod8423012012-10-26 13:37:29 +02004763 if (mem_cgroup_is_root(memcg))
4764 return -EINVAL;
Michal Hockoc26251f2012-10-26 13:37:28 +02004765 css_get(&memcg->css);
4766 ret = mem_cgroup_force_empty(memcg);
4767 css_put(&memcg->css);
4768
4769 return ret;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004770}
4771
4772
Balbir Singh18f59ea2009-01-07 18:08:07 -08004773static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
4774{
4775 return mem_cgroup_from_cont(cont)->use_hierarchy;
4776}
4777
4778static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
4779 u64 val)
4780{
4781 int retval = 0;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004782 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
Balbir Singh18f59ea2009-01-07 18:08:07 -08004783 struct cgroup *parent = cont->parent;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004784 struct mem_cgroup *parent_memcg = NULL;
Balbir Singh18f59ea2009-01-07 18:08:07 -08004785
4786 if (parent)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004787 parent_memcg = mem_cgroup_from_cont(parent);
Balbir Singh18f59ea2009-01-07 18:08:07 -08004788
4789 cgroup_lock();
Glauber Costa567fb432012-07-31 16:43:07 -07004790
4791 if (memcg->use_hierarchy == val)
4792 goto out;
4793
Balbir Singh18f59ea2009-01-07 18:08:07 -08004794 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02004795 * If parent's use_hierarchy is set, we can't make any modifications
Balbir Singh18f59ea2009-01-07 18:08:07 -08004796 * in the child subtrees. If it is unset, then the change can
4797 * occur, provided the current cgroup has no children.
4798 *
4799 * For the root cgroup, parent_mem is NULL, we allow value to be
4800 * set if there are no children.
4801 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004802 if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
Balbir Singh18f59ea2009-01-07 18:08:07 -08004803 (val == 1 || val == 0)) {
4804 if (list_empty(&cont->children))
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004805 memcg->use_hierarchy = val;
Balbir Singh18f59ea2009-01-07 18:08:07 -08004806 else
4807 retval = -EBUSY;
4808 } else
4809 retval = -EINVAL;
Glauber Costa567fb432012-07-31 16:43:07 -07004810
4811out:
Balbir Singh18f59ea2009-01-07 18:08:07 -08004812 cgroup_unlock();
4813
4814 return retval;
4815}
4816
Balbir Singh0c3e73e2009-09-23 15:56:42 -07004817
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004818static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
Johannes Weiner7a159cc2011-03-23 16:42:38 -07004819 enum mem_cgroup_stat_index idx)
Balbir Singh0c3e73e2009-09-23 15:56:42 -07004820{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07004821 struct mem_cgroup *iter;
Johannes Weiner7a159cc2011-03-23 16:42:38 -07004822 long val = 0;
Balbir Singh0c3e73e2009-09-23 15:56:42 -07004823
Johannes Weiner7a159cc2011-03-23 16:42:38 -07004824 /* Per-cpu values can be negative, use a signed accumulator */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004825 for_each_mem_cgroup_tree(iter, memcg)
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07004826 val += mem_cgroup_read_stat(iter, idx);
4827
4828 if (val < 0) /* race ? */
4829 val = 0;
4830 return val;
Balbir Singh0c3e73e2009-09-23 15:56:42 -07004831}
4832
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004833static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08004834{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07004835 u64 val;
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08004836
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004837 if (!mem_cgroup_is_root(memcg)) {
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08004838 if (!swap)
Glauber Costa65c64ce2011-12-22 01:02:27 +00004839 return res_counter_read_u64(&memcg->res, RES_USAGE);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08004840 else
Glauber Costa65c64ce2011-12-22 01:02:27 +00004841 return res_counter_read_u64(&memcg->memsw, RES_USAGE);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08004842 }
4843
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004844 val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
4845 val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08004846
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07004847 if (swap)
Kamezawa Hiroyukibff6bb82012-07-31 16:41:38 -07004848 val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08004849
4850 return val << PAGE_SHIFT;
4851}
4852
Tejun Heoaf36f902012-04-01 12:09:55 -07004853static ssize_t mem_cgroup_read(struct cgroup *cont, struct cftype *cft,
4854 struct file *file, char __user *buf,
4855 size_t nbytes, loff_t *ppos)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004856{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004857 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
Tejun Heoaf36f902012-04-01 12:09:55 -07004858 char str[64];
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08004859 u64 val;
Glauber Costa86ae53e2012-12-18 14:21:45 -08004860 int name, len;
4861 enum res_type type;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004862
4863 type = MEMFILE_TYPE(cft->private);
4864 name = MEMFILE_ATTR(cft->private);
Tejun Heoaf36f902012-04-01 12:09:55 -07004865
4866 if (!do_swap_account && type == _MEMSWAP)
4867 return -EOPNOTSUPP;
4868
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004869 switch (type) {
4870 case _MEM:
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08004871 if (name == RES_USAGE)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004872 val = mem_cgroup_usage(memcg, false);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08004873 else
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004874 val = res_counter_read_u64(&memcg->res, name);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004875 break;
4876 case _MEMSWAP:
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08004877 if (name == RES_USAGE)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004878 val = mem_cgroup_usage(memcg, true);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08004879 else
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004880 val = res_counter_read_u64(&memcg->memsw, name);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004881 break;
Glauber Costa510fc4e2012-12-18 14:21:47 -08004882 case _KMEM:
4883 val = res_counter_read_u64(&memcg->kmem, name);
4884 break;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004885 default:
4886 BUG();
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004887 }
Tejun Heoaf36f902012-04-01 12:09:55 -07004888
4889 len = scnprintf(str, sizeof(str), "%llu\n", (unsigned long long)val);
4890 return simple_read_from_buffer(buf, nbytes, ppos, str, len);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08004891}
Glauber Costa510fc4e2012-12-18 14:21:47 -08004892
4893static int memcg_update_kmem_limit(struct cgroup *cont, u64 val)
4894{
4895 int ret = -EINVAL;
4896#ifdef CONFIG_MEMCG_KMEM
Glauber Costaa8964b92012-12-18 14:22:09 -08004897 bool must_inc_static_branch = false;
4898
Glauber Costa510fc4e2012-12-18 14:21:47 -08004899 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
4900 /*
4901 * For simplicity, we won't allow this to be disabled. It also can't
4902 * be changed if the cgroup has children already, or if tasks had
4903 * already joined.
4904 *
4905 * If tasks join before we set the limit, a person looking at
4906 * kmem.usage_in_bytes will have no way to determine when it took
4907 * place, which makes the value quite meaningless.
4908 *
4909 * After it first became limited, changes in the value of the limit are
4910 * of course permitted.
4911 *
4912 * Taking the cgroup_lock is really offensive, but it is so far the only
4913 * way to guarantee that no children will appear. There are plenty of
4914 * other offenders, and they should all go away. Fine grained locking
4915 * is probably the way to go here. When we are fully hierarchical, we
4916 * can also get rid of the use_hierarchy check.
4917 */
4918 cgroup_lock();
4919 mutex_lock(&set_limit_mutex);
4920 if (!memcg->kmem_account_flags && val != RESOURCE_MAX) {
4921 if (cgroup_task_count(cont) || (memcg->use_hierarchy &&
4922 !list_empty(&cont->children))) {
4923 ret = -EBUSY;
4924 goto out;
4925 }
4926 ret = res_counter_set_limit(&memcg->kmem, val);
4927 VM_BUG_ON(ret);
4928
Glauber Costa55007d82012-12-18 14:22:38 -08004929 ret = memcg_update_cache_sizes(memcg);
4930 if (ret) {
4931 res_counter_set_limit(&memcg->kmem, RESOURCE_MAX);
4932 goto out;
4933 }
Glauber Costaa8964b92012-12-18 14:22:09 -08004934 must_inc_static_branch = true;
Glauber Costa7de37682012-12-18 14:22:07 -08004935 /*
4936 * kmem charges can outlive the cgroup. In the case of slab
4937 * pages, for instance, a page contain objects from various
4938 * processes, so it is unfeasible to migrate them away. We
4939 * need to reference count the memcg because of that.
4940 */
4941 mem_cgroup_get(memcg);
Glauber Costa510fc4e2012-12-18 14:21:47 -08004942 } else
4943 ret = res_counter_set_limit(&memcg->kmem, val);
4944out:
4945 mutex_unlock(&set_limit_mutex);
4946 cgroup_unlock();
Glauber Costaa8964b92012-12-18 14:22:09 -08004947
4948 /*
4949 * We are by now familiar with the fact that we can't inc the static
4950 * branch inside cgroup_lock. See disarm functions for details. A
4951 * worker here is overkill, but also wrong: After the limit is set, we
4952 * must start accounting right away. Since this operation can't fail,
4953 * we can safely defer it to here - no rollback will be needed.
4954 *
4955 * The boolean used to control this is also safe, because
4956 * KMEM_ACCOUNTED_ACTIVATED guarantees that only one process will be
4957 * able to set it to true;
4958 */
4959 if (must_inc_static_branch) {
4960 static_key_slow_inc(&memcg_kmem_enabled_key);
4961 /*
4962 * setting the active bit after the inc will guarantee no one
4963 * starts accounting before all call sites are patched
4964 */
4965 memcg_kmem_set_active(memcg);
4966 }
4967
Glauber Costa510fc4e2012-12-18 14:21:47 -08004968#endif
4969 return ret;
4970}
4971
Glauber Costa55007d82012-12-18 14:22:38 -08004972static int memcg_propagate_kmem(struct mem_cgroup *memcg)
Glauber Costa510fc4e2012-12-18 14:21:47 -08004973{
Glauber Costa55007d82012-12-18 14:22:38 -08004974 int ret = 0;
Glauber Costa510fc4e2012-12-18 14:21:47 -08004975 struct mem_cgroup *parent = parent_mem_cgroup(memcg);
4976 if (!parent)
Glauber Costa55007d82012-12-18 14:22:38 -08004977 goto out;
4978
Glauber Costa510fc4e2012-12-18 14:21:47 -08004979 memcg->kmem_account_flags = parent->kmem_account_flags;
Glauber Costa7de37682012-12-18 14:22:07 -08004980#ifdef CONFIG_MEMCG_KMEM
Glauber Costaa8964b92012-12-18 14:22:09 -08004981 /*
4982 * When that happen, we need to disable the static branch only on those
4983 * memcgs that enabled it. To achieve this, we would be forced to
4984 * complicate the code by keeping track of which memcgs were the ones
4985 * that actually enabled limits, and which ones got it from its
4986 * parents.
4987 *
4988 * It is a lot simpler just to do static_key_slow_inc() on every child
4989 * that is accounted.
4990 */
Glauber Costa55007d82012-12-18 14:22:38 -08004991 if (!memcg_kmem_is_active(memcg))
4992 goto out;
4993
4994 /*
4995 * destroy(), called if we fail, will issue static_key_slow_inc() and
4996 * mem_cgroup_put() if kmem is enabled. We have to either call them
4997 * unconditionally, or clear the KMEM_ACTIVE flag. I personally find
4998 * this more consistent, since it always leads to the same destroy path
4999 */
5000 mem_cgroup_get(memcg);
5001 static_key_slow_inc(&memcg_kmem_enabled_key);
5002
5003 mutex_lock(&set_limit_mutex);
5004 ret = memcg_update_cache_sizes(memcg);
5005 mutex_unlock(&set_limit_mutex);
Glauber Costa7de37682012-12-18 14:22:07 -08005006#endif
Glauber Costa55007d82012-12-18 14:22:38 -08005007out:
5008 return ret;
Glauber Costa510fc4e2012-12-18 14:21:47 -08005009}
5010
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005011/*
5012 * The user of this function is...
5013 * RES_LIMIT.
5014 */
Paul Menage856c13a2008-07-25 01:47:04 -07005015static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
5016 const char *buffer)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005017{
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005018 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
Glauber Costa86ae53e2012-12-18 14:21:45 -08005019 enum res_type type;
5020 int name;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005021 unsigned long long val;
5022 int ret;
5023
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005024 type = MEMFILE_TYPE(cft->private);
5025 name = MEMFILE_ATTR(cft->private);
Tejun Heoaf36f902012-04-01 12:09:55 -07005026
5027 if (!do_swap_account && type == _MEMSWAP)
5028 return -EOPNOTSUPP;
5029
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005030 switch (name) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005031 case RES_LIMIT:
Balbir Singh4b3bde42009-09-23 15:56:32 -07005032 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
5033 ret = -EINVAL;
5034 break;
5035 }
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005036 /* This function does all necessary parse...reuse it */
5037 ret = res_counter_memparse_write_strategy(buffer, &val);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005038 if (ret)
5039 break;
5040 if (type == _MEM)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005041 ret = mem_cgroup_resize_limit(memcg, val);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005042 else if (type == _MEMSWAP)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005043 ret = mem_cgroup_resize_memsw_limit(memcg, val);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005044 else if (type == _KMEM)
5045 ret = memcg_update_kmem_limit(cont, val);
5046 else
5047 return -EINVAL;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005048 break;
Balbir Singh296c81d2009-09-23 15:56:36 -07005049 case RES_SOFT_LIMIT:
5050 ret = res_counter_memparse_write_strategy(buffer, &val);
5051 if (ret)
5052 break;
5053 /*
5054 * For memsw, soft limits are hard to implement in terms
5055 * of semantics, for now, we support soft limits for
5056 * control without swap
5057 */
5058 if (type == _MEM)
5059 ret = res_counter_set_soft_limit(&memcg->res, val);
5060 else
5061 ret = -EINVAL;
5062 break;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005063 default:
5064 ret = -EINVAL; /* should be BUG() ? */
5065 break;
5066 }
5067 return ret;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005068}
5069
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005070static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
5071 unsigned long long *mem_limit, unsigned long long *memsw_limit)
5072{
5073 struct cgroup *cgroup;
5074 unsigned long long min_limit, min_memsw_limit, tmp;
5075
5076 min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
5077 min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
5078 cgroup = memcg->css.cgroup;
5079 if (!memcg->use_hierarchy)
5080 goto out;
5081
5082 while (cgroup->parent) {
5083 cgroup = cgroup->parent;
5084 memcg = mem_cgroup_from_cont(cgroup);
5085 if (!memcg->use_hierarchy)
5086 break;
5087 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
5088 min_limit = min(min_limit, tmp);
5089 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
5090 min_memsw_limit = min(min_memsw_limit, tmp);
5091 }
5092out:
5093 *mem_limit = min_limit;
5094 *memsw_limit = min_memsw_limit;
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005095}
5096
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07005097static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07005098{
Tejun Heoaf36f902012-04-01 12:09:55 -07005099 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
Glauber Costa86ae53e2012-12-18 14:21:45 -08005100 int name;
5101 enum res_type type;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07005102
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005103 type = MEMFILE_TYPE(event);
5104 name = MEMFILE_ATTR(event);
Tejun Heoaf36f902012-04-01 12:09:55 -07005105
5106 if (!do_swap_account && type == _MEMSWAP)
5107 return -EOPNOTSUPP;
5108
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005109 switch (name) {
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07005110 case RES_MAX_USAGE:
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005111 if (type == _MEM)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005112 res_counter_reset_max(&memcg->res);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005113 else if (type == _MEMSWAP)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005114 res_counter_reset_max(&memcg->memsw);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005115 else if (type == _KMEM)
5116 res_counter_reset_max(&memcg->kmem);
5117 else
5118 return -EINVAL;
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07005119 break;
5120 case RES_FAILCNT:
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005121 if (type == _MEM)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005122 res_counter_reset_failcnt(&memcg->res);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005123 else if (type == _MEMSWAP)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005124 res_counter_reset_failcnt(&memcg->memsw);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005125 else if (type == _KMEM)
5126 res_counter_reset_failcnt(&memcg->kmem);
5127 else
5128 return -EINVAL;
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07005129 break;
5130 }
Balbir Singhf64c3f52009-09-23 15:56:37 -07005131
Pavel Emelyanov85cc59d2008-04-29 01:00:20 -07005132 return 0;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07005133}
5134
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005135static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
5136 struct cftype *cft)
5137{
5138 return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
5139}
5140
Daisuke Nishimura02491442010-03-10 15:22:17 -08005141#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005142static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
5143 struct cftype *cft, u64 val)
5144{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005145 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005146
5147 if (val >= (1 << NR_MOVE_TYPE))
5148 return -EINVAL;
5149 /*
5150 * We check this value several times in both in can_attach() and
5151 * attach(), so we need cgroup lock to prevent this value from being
5152 * inconsistent.
5153 */
5154 cgroup_lock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005155 memcg->move_charge_at_immigrate = val;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005156 cgroup_unlock();
5157
5158 return 0;
5159}
Daisuke Nishimura02491442010-03-10 15:22:17 -08005160#else
5161static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
5162 struct cftype *cft, u64 val)
5163{
5164 return -ENOSYS;
5165}
5166#endif
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005167
Ying Han406eb0c2011-05-26 16:25:37 -07005168#ifdef CONFIG_NUMA
Wanpeng Liab215882012-07-31 16:43:09 -07005169static int memcg_numa_stat_show(struct cgroup *cont, struct cftype *cft,
Johannes Weinerfada52c2012-05-29 15:07:06 -07005170 struct seq_file *m)
Ying Han406eb0c2011-05-26 16:25:37 -07005171{
5172 int nid;
5173 unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
5174 unsigned long node_nr;
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005175 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
Ying Han406eb0c2011-05-26 16:25:37 -07005176
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005177 total_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL);
Ying Han406eb0c2011-05-26 16:25:37 -07005178 seq_printf(m, "total=%lu", total_nr);
Lai Jiangshan31aaea42012-12-12 13:51:27 -08005179 for_each_node_state(nid, N_MEMORY) {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005180 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL);
Ying Han406eb0c2011-05-26 16:25:37 -07005181 seq_printf(m, " N%d=%lu", nid, node_nr);
5182 }
5183 seq_putc(m, '\n');
5184
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005185 file_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_FILE);
Ying Han406eb0c2011-05-26 16:25:37 -07005186 seq_printf(m, "file=%lu", file_nr);
Lai Jiangshan31aaea42012-12-12 13:51:27 -08005187 for_each_node_state(nid, N_MEMORY) {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005188 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07005189 LRU_ALL_FILE);
Ying Han406eb0c2011-05-26 16:25:37 -07005190 seq_printf(m, " N%d=%lu", nid, node_nr);
5191 }
5192 seq_putc(m, '\n');
5193
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005194 anon_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_ANON);
Ying Han406eb0c2011-05-26 16:25:37 -07005195 seq_printf(m, "anon=%lu", anon_nr);
Lai Jiangshan31aaea42012-12-12 13:51:27 -08005196 for_each_node_state(nid, N_MEMORY) {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005197 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07005198 LRU_ALL_ANON);
Ying Han406eb0c2011-05-26 16:25:37 -07005199 seq_printf(m, " N%d=%lu", nid, node_nr);
5200 }
5201 seq_putc(m, '\n');
5202
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005203 unevictable_nr = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
Ying Han406eb0c2011-05-26 16:25:37 -07005204 seq_printf(m, "unevictable=%lu", unevictable_nr);
Lai Jiangshan31aaea42012-12-12 13:51:27 -08005205 for_each_node_state(nid, N_MEMORY) {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005206 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07005207 BIT(LRU_UNEVICTABLE));
Ying Han406eb0c2011-05-26 16:25:37 -07005208 seq_printf(m, " N%d=%lu", nid, node_nr);
5209 }
5210 seq_putc(m, '\n');
5211 return 0;
5212}
5213#endif /* CONFIG_NUMA */
5214
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005215static const char * const mem_cgroup_lru_names[] = {
5216 "inactive_anon",
5217 "active_anon",
5218 "inactive_file",
5219 "active_file",
5220 "unevictable",
5221};
5222
5223static inline void mem_cgroup_lru_names_not_uptodate(void)
5224{
5225 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
5226}
5227
Wanpeng Liab215882012-07-31 16:43:09 -07005228static int memcg_stat_show(struct cgroup *cont, struct cftype *cft,
Johannes Weiner78ccf5b2012-05-29 15:07:06 -07005229 struct seq_file *m)
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08005230{
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005231 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005232 struct mem_cgroup *mi;
5233 unsigned int i;
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08005234
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005235 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
Kamezawa Hiroyukibff6bb82012-07-31 16:41:38 -07005236 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07005237 continue;
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005238 seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
5239 mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07005240 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005241
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005242 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
5243 seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
5244 mem_cgroup_read_events(memcg, i));
5245
5246 for (i = 0; i < NR_LRU_LISTS; i++)
5247 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
5248 mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
5249
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07005250 /* Hierarchical information */
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005251 {
5252 unsigned long long limit, memsw_limit;
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005253 memcg_get_hierarchical_limit(memcg, &limit, &memsw_limit);
Johannes Weiner78ccf5b2012-05-29 15:07:06 -07005254 seq_printf(m, "hierarchical_memory_limit %llu\n", limit);
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005255 if (do_swap_account)
Johannes Weiner78ccf5b2012-05-29 15:07:06 -07005256 seq_printf(m, "hierarchical_memsw_limit %llu\n",
5257 memsw_limit);
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005258 }
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005259
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005260 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
5261 long long val = 0;
5262
Kamezawa Hiroyukibff6bb82012-07-31 16:41:38 -07005263 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07005264 continue;
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005265 for_each_mem_cgroup_tree(mi, memcg)
5266 val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
5267 seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
5268 }
5269
5270 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
5271 unsigned long long val = 0;
5272
5273 for_each_mem_cgroup_tree(mi, memcg)
5274 val += mem_cgroup_read_events(mi, i);
5275 seq_printf(m, "total_%s %llu\n",
5276 mem_cgroup_events_names[i], val);
5277 }
5278
5279 for (i = 0; i < NR_LRU_LISTS; i++) {
5280 unsigned long long val = 0;
5281
5282 for_each_mem_cgroup_tree(mi, memcg)
5283 val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
5284 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07005285 }
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07005286
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005287#ifdef CONFIG_DEBUG_VM
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005288 {
5289 int nid, zid;
5290 struct mem_cgroup_per_zone *mz;
Hugh Dickins89abfab2012-05-29 15:06:53 -07005291 struct zone_reclaim_stat *rstat;
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005292 unsigned long recent_rotated[2] = {0, 0};
5293 unsigned long recent_scanned[2] = {0, 0};
5294
5295 for_each_online_node(nid)
5296 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005297 mz = mem_cgroup_zoneinfo(memcg, nid, zid);
Hugh Dickins89abfab2012-05-29 15:06:53 -07005298 rstat = &mz->lruvec.reclaim_stat;
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005299
Hugh Dickins89abfab2012-05-29 15:06:53 -07005300 recent_rotated[0] += rstat->recent_rotated[0];
5301 recent_rotated[1] += rstat->recent_rotated[1];
5302 recent_scanned[0] += rstat->recent_scanned[0];
5303 recent_scanned[1] += rstat->recent_scanned[1];
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005304 }
Johannes Weiner78ccf5b2012-05-29 15:07:06 -07005305 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
5306 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
5307 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
5308 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005309 }
5310#endif
5311
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08005312 return 0;
5313}
5314
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005315static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
5316{
5317 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5318
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07005319 return mem_cgroup_swappiness(memcg);
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005320}
5321
5322static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
5323 u64 val)
5324{
5325 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5326 struct mem_cgroup *parent;
Li Zefan068b38c2009-01-15 13:51:26 -08005327
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005328 if (val > 100)
5329 return -EINVAL;
5330
5331 if (cgrp->parent == NULL)
5332 return -EINVAL;
5333
5334 parent = mem_cgroup_from_cont(cgrp->parent);
Li Zefan068b38c2009-01-15 13:51:26 -08005335
5336 cgroup_lock();
5337
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005338 /* If under hierarchy, only empty-root can set this value */
5339 if ((parent->use_hierarchy) ||
Li Zefan068b38c2009-01-15 13:51:26 -08005340 (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
5341 cgroup_unlock();
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005342 return -EINVAL;
Li Zefan068b38c2009-01-15 13:51:26 -08005343 }
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005344
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005345 memcg->swappiness = val;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005346
Li Zefan068b38c2009-01-15 13:51:26 -08005347 cgroup_unlock();
5348
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005349 return 0;
5350}
5351
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005352static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
5353{
5354 struct mem_cgroup_threshold_ary *t;
5355 u64 usage;
5356 int i;
5357
5358 rcu_read_lock();
5359 if (!swap)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005360 t = rcu_dereference(memcg->thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005361 else
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005362 t = rcu_dereference(memcg->memsw_thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005363
5364 if (!t)
5365 goto unlock;
5366
5367 usage = mem_cgroup_usage(memcg, swap);
5368
5369 /*
Sha Zhengju748dad32012-05-29 15:06:57 -07005370 * current_threshold points to threshold just below or equal to usage.
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005371 * If it's not true, a threshold was crossed after last
5372 * call of __mem_cgroup_threshold().
5373 */
Phil Carmody5407a562010-05-26 14:42:42 -07005374 i = t->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005375
5376 /*
5377 * Iterate backward over array of thresholds starting from
5378 * current_threshold and check if a threshold is crossed.
5379 * If none of thresholds below usage is crossed, we read
5380 * only one element of the array here.
5381 */
5382 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
5383 eventfd_signal(t->entries[i].eventfd, 1);
5384
5385 /* i = current_threshold + 1 */
5386 i++;
5387
5388 /*
5389 * Iterate forward over array of thresholds starting from
5390 * current_threshold+1 and check if a threshold is crossed.
5391 * If none of thresholds above usage is crossed, we read
5392 * only one element of the array here.
5393 */
5394 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
5395 eventfd_signal(t->entries[i].eventfd, 1);
5396
5397 /* Update current_threshold */
Phil Carmody5407a562010-05-26 14:42:42 -07005398 t->current_threshold = i - 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005399unlock:
5400 rcu_read_unlock();
5401}
5402
5403static void mem_cgroup_threshold(struct mem_cgroup *memcg)
5404{
Kirill A. Shutemovad4ca5f2010-10-07 12:59:27 -07005405 while (memcg) {
5406 __mem_cgroup_threshold(memcg, false);
5407 if (do_swap_account)
5408 __mem_cgroup_threshold(memcg, true);
5409
5410 memcg = parent_mem_cgroup(memcg);
5411 }
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005412}
5413
5414static int compare_thresholds(const void *a, const void *b)
5415{
5416 const struct mem_cgroup_threshold *_a = a;
5417 const struct mem_cgroup_threshold *_b = b;
5418
5419 return _a->threshold - _b->threshold;
5420}
5421
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005422static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005423{
5424 struct mem_cgroup_eventfd_list *ev;
5425
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005426 list_for_each_entry(ev, &memcg->oom_notify, list)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005427 eventfd_signal(ev->eventfd, 1);
5428 return 0;
5429}
5430
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005431static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005432{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07005433 struct mem_cgroup *iter;
5434
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005435 for_each_mem_cgroup_tree(iter, memcg)
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07005436 mem_cgroup_oom_notify_cb(iter);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005437}
5438
5439static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
5440 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005441{
5442 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005443 struct mem_cgroup_thresholds *thresholds;
5444 struct mem_cgroup_threshold_ary *new;
Glauber Costa86ae53e2012-12-18 14:21:45 -08005445 enum res_type type = MEMFILE_TYPE(cft->private);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005446 u64 threshold, usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005447 int i, size, ret;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005448
5449 ret = res_counter_memparse_write_strategy(args, &threshold);
5450 if (ret)
5451 return ret;
5452
5453 mutex_lock(&memcg->thresholds_lock);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005454
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005455 if (type == _MEM)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005456 thresholds = &memcg->thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005457 else if (type == _MEMSWAP)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005458 thresholds = &memcg->memsw_thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005459 else
5460 BUG();
5461
5462 usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
5463
5464 /* Check if a threshold crossed before adding a new one */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005465 if (thresholds->primary)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005466 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
5467
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005468 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005469
5470 /* Allocate memory for new array of thresholds */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005471 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005472 GFP_KERNEL);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005473 if (!new) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005474 ret = -ENOMEM;
5475 goto unlock;
5476 }
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005477 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005478
5479 /* Copy thresholds (if any) to new array */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005480 if (thresholds->primary) {
5481 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005482 sizeof(struct mem_cgroup_threshold));
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005483 }
5484
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005485 /* Add new threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005486 new->entries[size - 1].eventfd = eventfd;
5487 new->entries[size - 1].threshold = threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005488
5489 /* Sort thresholds. Registering of new threshold isn't time-critical */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005490 sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005491 compare_thresholds, NULL);
5492
5493 /* Find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005494 new->current_threshold = -1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005495 for (i = 0; i < size; i++) {
Sha Zhengju748dad32012-05-29 15:06:57 -07005496 if (new->entries[i].threshold <= usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005497 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005498 * new->current_threshold will not be used until
5499 * rcu_assign_pointer(), so it's safe to increment
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005500 * it here.
5501 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005502 ++new->current_threshold;
Sha Zhengju748dad32012-05-29 15:06:57 -07005503 } else
5504 break;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005505 }
5506
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005507 /* Free old spare buffer and save old primary buffer as spare */
5508 kfree(thresholds->spare);
5509 thresholds->spare = thresholds->primary;
5510
5511 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005512
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07005513 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005514 synchronize_rcu();
5515
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005516unlock:
5517 mutex_unlock(&memcg->thresholds_lock);
5518
5519 return ret;
5520}
5521
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07005522static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005523 struct cftype *cft, struct eventfd_ctx *eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005524{
5525 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005526 struct mem_cgroup_thresholds *thresholds;
5527 struct mem_cgroup_threshold_ary *new;
Glauber Costa86ae53e2012-12-18 14:21:45 -08005528 enum res_type type = MEMFILE_TYPE(cft->private);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005529 u64 usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005530 int i, j, size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005531
5532 mutex_lock(&memcg->thresholds_lock);
5533 if (type == _MEM)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005534 thresholds = &memcg->thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005535 else if (type == _MEMSWAP)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005536 thresholds = &memcg->memsw_thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005537 else
5538 BUG();
5539
Anton Vorontsov371528c2012-02-24 05:14:46 +04005540 if (!thresholds->primary)
5541 goto unlock;
5542
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005543 usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
5544
5545 /* Check if a threshold crossed before removing */
5546 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
5547
5548 /* Calculate new number of threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005549 size = 0;
5550 for (i = 0; i < thresholds->primary->size; i++) {
5551 if (thresholds->primary->entries[i].eventfd != eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005552 size++;
5553 }
5554
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005555 new = thresholds->spare;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07005556
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005557 /* Set thresholds array to NULL if we don't have thresholds */
5558 if (!size) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005559 kfree(new);
5560 new = NULL;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07005561 goto swap_buffers;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005562 }
5563
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005564 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005565
5566 /* Copy thresholds and find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005567 new->current_threshold = -1;
5568 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
5569 if (thresholds->primary->entries[i].eventfd == eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005570 continue;
5571
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005572 new->entries[j] = thresholds->primary->entries[i];
Sha Zhengju748dad32012-05-29 15:06:57 -07005573 if (new->entries[j].threshold <= usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005574 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005575 * new->current_threshold will not be used
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005576 * until rcu_assign_pointer(), so it's safe to increment
5577 * it here.
5578 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005579 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005580 }
5581 j++;
5582 }
5583
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07005584swap_buffers:
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005585 /* Swap primary and spare array */
5586 thresholds->spare = thresholds->primary;
Sha Zhengju8c757762012-05-10 13:01:45 -07005587 /* If all events are unregistered, free the spare array */
5588 if (!new) {
5589 kfree(thresholds->spare);
5590 thresholds->spare = NULL;
5591 }
5592
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005593 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005594
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07005595 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005596 synchronize_rcu();
Anton Vorontsov371528c2012-02-24 05:14:46 +04005597unlock:
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005598 mutex_unlock(&memcg->thresholds_lock);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005599}
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08005600
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005601static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
5602 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
5603{
5604 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
5605 struct mem_cgroup_eventfd_list *event;
Glauber Costa86ae53e2012-12-18 14:21:45 -08005606 enum res_type type = MEMFILE_TYPE(cft->private);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005607
5608 BUG_ON(type != _OOM_TYPE);
5609 event = kmalloc(sizeof(*event), GFP_KERNEL);
5610 if (!event)
5611 return -ENOMEM;
5612
Michal Hocko1af8efe2011-07-26 16:08:24 -07005613 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005614
5615 event->eventfd = eventfd;
5616 list_add(&event->list, &memcg->oom_notify);
5617
5618 /* already in OOM ? */
Michal Hocko79dfdac2011-07-26 16:08:23 -07005619 if (atomic_read(&memcg->under_oom))
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005620 eventfd_signal(eventfd, 1);
Michal Hocko1af8efe2011-07-26 16:08:24 -07005621 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005622
5623 return 0;
5624}
5625
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07005626static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005627 struct cftype *cft, struct eventfd_ctx *eventfd)
5628{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005629 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005630 struct mem_cgroup_eventfd_list *ev, *tmp;
Glauber Costa86ae53e2012-12-18 14:21:45 -08005631 enum res_type type = MEMFILE_TYPE(cft->private);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005632
5633 BUG_ON(type != _OOM_TYPE);
5634
Michal Hocko1af8efe2011-07-26 16:08:24 -07005635 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005636
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005637 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005638 if (ev->eventfd == eventfd) {
5639 list_del(&ev->list);
5640 kfree(ev);
5641 }
5642 }
5643
Michal Hocko1af8efe2011-07-26 16:08:24 -07005644 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005645}
5646
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005647static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
5648 struct cftype *cft, struct cgroup_map_cb *cb)
5649{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005650 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005651
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005652 cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005653
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005654 if (atomic_read(&memcg->under_oom))
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005655 cb->fill(cb, "under_oom", 1);
5656 else
5657 cb->fill(cb, "under_oom", 0);
5658 return 0;
5659}
5660
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005661static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
5662 struct cftype *cft, u64 val)
5663{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005664 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005665 struct mem_cgroup *parent;
5666
5667 /* cannot set to root cgroup and only 0 and 1 are allowed */
5668 if (!cgrp->parent || !((val == 0) || (val == 1)))
5669 return -EINVAL;
5670
5671 parent = mem_cgroup_from_cont(cgrp->parent);
5672
5673 cgroup_lock();
5674 /* oom-kill-disable is a flag for subhierarchy. */
5675 if ((parent->use_hierarchy) ||
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005676 (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005677 cgroup_unlock();
5678 return -EINVAL;
5679 }
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005680 memcg->oom_kill_disable = val;
KAMEZAWA Hiroyuki4d845eb2010-06-29 15:05:18 -07005681 if (!val)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005682 memcg_oom_recover(memcg);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005683 cgroup_unlock();
5684 return 0;
5685}
5686
Andrew Mortonc255a452012-07-31 16:43:02 -07005687#ifdef CONFIG_MEMCG_KMEM
Glauber Costacbe128e32012-04-09 19:36:34 -03005688static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
Glauber Costae5671df2011-12-11 21:47:01 +00005689{
Glauber Costa55007d82012-12-18 14:22:38 -08005690 int ret;
5691
Glauber Costa2633d7a2012-12-18 14:22:34 -08005692 memcg->kmemcg_id = -1;
Glauber Costa55007d82012-12-18 14:22:38 -08005693 ret = memcg_propagate_kmem(memcg);
5694 if (ret)
5695 return ret;
Glauber Costa2633d7a2012-12-18 14:22:34 -08005696
Glauber Costa1d62e432012-04-09 19:36:33 -03005697 return mem_cgroup_sockets_init(memcg, ss);
Glauber Costae5671df2011-12-11 21:47:01 +00005698};
5699
Glauber Costa1d62e432012-04-09 19:36:33 -03005700static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
Glauber Costad1a4c0b2011-12-11 21:47:04 +00005701{
Glauber Costa1d62e432012-04-09 19:36:33 -03005702 mem_cgroup_sockets_destroy(memcg);
Glauber Costa7de37682012-12-18 14:22:07 -08005703
5704 memcg_kmem_mark_dead(memcg);
5705
5706 if (res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0)
5707 return;
5708
5709 /*
5710 * Charges already down to 0, undo mem_cgroup_get() done in the charge
5711 * path here, being careful not to race with memcg_uncharge_kmem: it is
5712 * possible that the charges went down to 0 between mark_dead and the
5713 * res_counter read, so in that case, we don't need the put
5714 */
5715 if (memcg_kmem_test_and_clear_dead(memcg))
5716 mem_cgroup_put(memcg);
Glauber Costad1a4c0b2011-12-11 21:47:04 +00005717}
Glauber Costae5671df2011-12-11 21:47:01 +00005718#else
Glauber Costacbe128e32012-04-09 19:36:34 -03005719static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
Glauber Costae5671df2011-12-11 21:47:01 +00005720{
5721 return 0;
5722}
Glauber Costad1a4c0b2011-12-11 21:47:04 +00005723
Glauber Costa1d62e432012-04-09 19:36:33 -03005724static void kmem_cgroup_destroy(struct mem_cgroup *memcg)
Glauber Costad1a4c0b2011-12-11 21:47:04 +00005725{
5726}
Glauber Costae5671df2011-12-11 21:47:01 +00005727#endif
5728
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005729static struct cftype mem_cgroup_files[] = {
5730 {
Balbir Singh0eea1032008-02-07 00:13:57 -08005731 .name = "usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005732 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
Tejun Heoaf36f902012-04-01 12:09:55 -07005733 .read = mem_cgroup_read,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005734 .register_event = mem_cgroup_usage_register_event,
5735 .unregister_event = mem_cgroup_usage_unregister_event,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005736 },
5737 {
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07005738 .name = "max_usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005739 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07005740 .trigger = mem_cgroup_reset,
Tejun Heoaf36f902012-04-01 12:09:55 -07005741 .read = mem_cgroup_read,
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07005742 },
5743 {
Balbir Singh0eea1032008-02-07 00:13:57 -08005744 .name = "limit_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005745 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
Paul Menage856c13a2008-07-25 01:47:04 -07005746 .write_string = mem_cgroup_write,
Tejun Heoaf36f902012-04-01 12:09:55 -07005747 .read = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005748 },
5749 {
Balbir Singh296c81d2009-09-23 15:56:36 -07005750 .name = "soft_limit_in_bytes",
5751 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
5752 .write_string = mem_cgroup_write,
Tejun Heoaf36f902012-04-01 12:09:55 -07005753 .read = mem_cgroup_read,
Balbir Singh296c81d2009-09-23 15:56:36 -07005754 },
5755 {
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005756 .name = "failcnt",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005757 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07005758 .trigger = mem_cgroup_reset,
Tejun Heoaf36f902012-04-01 12:09:55 -07005759 .read = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005760 },
Balbir Singh8697d332008-02-07 00:13:59 -08005761 {
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08005762 .name = "stat",
Wanpeng Liab215882012-07-31 16:43:09 -07005763 .read_seq_string = memcg_stat_show,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08005764 },
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08005765 {
5766 .name = "force_empty",
5767 .trigger = mem_cgroup_force_empty_write,
5768 },
Balbir Singh18f59ea2009-01-07 18:08:07 -08005769 {
5770 .name = "use_hierarchy",
5771 .write_u64 = mem_cgroup_hierarchy_write,
5772 .read_u64 = mem_cgroup_hierarchy_read,
5773 },
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005774 {
5775 .name = "swappiness",
5776 .read_u64 = mem_cgroup_swappiness_read,
5777 .write_u64 = mem_cgroup_swappiness_write,
5778 },
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005779 {
5780 .name = "move_charge_at_immigrate",
5781 .read_u64 = mem_cgroup_move_charge_read,
5782 .write_u64 = mem_cgroup_move_charge_write,
5783 },
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005784 {
5785 .name = "oom_control",
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005786 .read_map = mem_cgroup_oom_control_read,
5787 .write_u64 = mem_cgroup_oom_control_write,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005788 .register_event = mem_cgroup_oom_register_event,
5789 .unregister_event = mem_cgroup_oom_unregister_event,
5790 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
5791 },
Ying Han406eb0c2011-05-26 16:25:37 -07005792#ifdef CONFIG_NUMA
5793 {
5794 .name = "numa_stat",
Wanpeng Liab215882012-07-31 16:43:09 -07005795 .read_seq_string = memcg_numa_stat_show,
Ying Han406eb0c2011-05-26 16:25:37 -07005796 },
5797#endif
Andrew Mortonc255a452012-07-31 16:43:02 -07005798#ifdef CONFIG_MEMCG_SWAP
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005799 {
5800 .name = "memsw.usage_in_bytes",
5801 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
Tejun Heoaf36f902012-04-01 12:09:55 -07005802 .read = mem_cgroup_read,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005803 .register_event = mem_cgroup_usage_register_event,
5804 .unregister_event = mem_cgroup_usage_unregister_event,
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005805 },
5806 {
5807 .name = "memsw.max_usage_in_bytes",
5808 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
5809 .trigger = mem_cgroup_reset,
Tejun Heoaf36f902012-04-01 12:09:55 -07005810 .read = mem_cgroup_read,
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005811 },
5812 {
5813 .name = "memsw.limit_in_bytes",
5814 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
5815 .write_string = mem_cgroup_write,
Tejun Heoaf36f902012-04-01 12:09:55 -07005816 .read = mem_cgroup_read,
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005817 },
5818 {
5819 .name = "memsw.failcnt",
5820 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
5821 .trigger = mem_cgroup_reset,
Tejun Heoaf36f902012-04-01 12:09:55 -07005822 .read = mem_cgroup_read,
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005823 },
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005824#endif
Glauber Costa510fc4e2012-12-18 14:21:47 -08005825#ifdef CONFIG_MEMCG_KMEM
5826 {
5827 .name = "kmem.limit_in_bytes",
5828 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
5829 .write_string = mem_cgroup_write,
5830 .read = mem_cgroup_read,
5831 },
5832 {
5833 .name = "kmem.usage_in_bytes",
5834 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
5835 .read = mem_cgroup_read,
5836 },
5837 {
5838 .name = "kmem.failcnt",
5839 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
5840 .trigger = mem_cgroup_reset,
5841 .read = mem_cgroup_read,
5842 },
5843 {
5844 .name = "kmem.max_usage_in_bytes",
5845 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
5846 .trigger = mem_cgroup_reset,
5847 .read = mem_cgroup_read,
5848 },
Glauber Costa749c5412012-12-18 14:23:01 -08005849#ifdef CONFIG_SLABINFO
5850 {
5851 .name = "kmem.slabinfo",
5852 .read_seq_string = mem_cgroup_slabinfo_read,
5853 },
5854#endif
Glauber Costa510fc4e2012-12-18 14:21:47 -08005855#endif
Tejun Heo6bc10342012-04-01 12:09:55 -07005856 { }, /* terminate */
Tejun Heoaf36f902012-04-01 12:09:55 -07005857};
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005858
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005859static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005860{
5861 struct mem_cgroup_per_node *pn;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005862 struct mem_cgroup_per_zone *mz;
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07005863 int zone, tmp = node;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005864 /*
5865 * This routine is called against possible nodes.
5866 * But it's BUG to call kmalloc() against offline node.
5867 *
5868 * TODO: this routine can waste much memory for nodes which will
5869 * never be onlined. It's better to use memory hotplug callback
5870 * function.
5871 */
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07005872 if (!node_state(node, N_NORMAL_MEMORY))
5873 tmp = -1;
Jesper Juhl17295c82011-01-13 15:47:42 -08005874 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005875 if (!pn)
5876 return 1;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005877
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005878 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
5879 mz = &pn->zoneinfo[zone];
Hugh Dickinsbea8c152012-11-16 14:14:54 -08005880 lruvec_init(&mz->lruvec);
Balbir Singhf64c3f52009-09-23 15:56:37 -07005881 mz->usage_in_excess = 0;
Balbir Singh4e416952009-09-23 15:56:39 -07005882 mz->on_tree = false;
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005883 mz->memcg = memcg;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005884 }
Igor Mammedov0a619e52011-11-02 13:38:21 -07005885 memcg->info.nodeinfo[node] = pn;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005886 return 0;
5887}
5888
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005889static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005890{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005891 kfree(memcg->info.nodeinfo[node]);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08005892}
5893
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07005894static struct mem_cgroup *mem_cgroup_alloc(void)
5895{
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005896 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08005897 int size = sizeof(struct mem_cgroup);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07005898
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -08005899 /* Can be very big if MAX_NUMNODES is very big */
Jan Blunckc8dad2b2009-01-07 18:07:53 -08005900 if (size < PAGE_SIZE)
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005901 memcg = kzalloc(size, GFP_KERNEL);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07005902 else
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005903 memcg = vzalloc(size);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07005904
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005905 if (!memcg)
Dan Carpentere7bbcdf2010-03-23 13:35:12 -07005906 return NULL;
5907
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005908 memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
5909 if (!memcg->stat)
Dan Carpenterd2e61b82010-11-11 14:05:12 -08005910 goto out_free;
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005911 spin_lock_init(&memcg->pcp_counter_lock);
5912 return memcg;
Dan Carpenterd2e61b82010-11-11 14:05:12 -08005913
5914out_free:
5915 if (size < PAGE_SIZE)
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005916 kfree(memcg);
Dan Carpenterd2e61b82010-11-11 14:05:12 -08005917 else
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005918 vfree(memcg);
Dan Carpenterd2e61b82010-11-11 14:05:12 -08005919 return NULL;
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07005920}
5921
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005922/*
Glauber Costac8b2a362012-12-18 14:22:13 -08005923 * At destroying mem_cgroup, references from swap_cgroup can remain.
5924 * (scanning all at force_empty is too costly...)
5925 *
5926 * Instead of clearing all references at force_empty, we remember
5927 * the number of reference from swap_cgroup and free mem_cgroup when
5928 * it goes down to 0.
5929 *
5930 * Removal of cgroup itself succeeds regardless of refs from swap.
Hugh Dickins59927fb2012-03-15 15:17:07 -07005931 */
Glauber Costac8b2a362012-12-18 14:22:13 -08005932
5933static void __mem_cgroup_free(struct mem_cgroup *memcg)
Hugh Dickins59927fb2012-03-15 15:17:07 -07005934{
Glauber Costac8b2a362012-12-18 14:22:13 -08005935 int node;
Glauber Costa3afe36b2012-05-29 15:07:10 -07005936 int size = sizeof(struct mem_cgroup);
Hugh Dickins59927fb2012-03-15 15:17:07 -07005937
Glauber Costac8b2a362012-12-18 14:22:13 -08005938 mem_cgroup_remove_from_trees(memcg);
5939 free_css_id(&mem_cgroup_subsys, &memcg->css);
5940
5941 for_each_node(node)
5942 free_mem_cgroup_per_zone_info(memcg, node);
5943
5944 free_percpu(memcg->stat);
5945
Glauber Costa3f134612012-05-29 15:07:11 -07005946 /*
5947 * We need to make sure that (at least for now), the jump label
5948 * destruction code runs outside of the cgroup lock. This is because
5949 * get_online_cpus(), which is called from the static_branch update,
5950 * can't be called inside the cgroup_lock. cpusets are the ones
5951 * enforcing this dependency, so if they ever change, we might as well.
5952 *
5953 * schedule_work() will guarantee this happens. Be careful if you need
5954 * to move this code around, and make sure it is outside
5955 * the cgroup_lock.
5956 */
Glauber Costaa8964b92012-12-18 14:22:09 -08005957 disarm_static_keys(memcg);
Glauber Costa3afe36b2012-05-29 15:07:10 -07005958 if (size < PAGE_SIZE)
5959 kfree(memcg);
5960 else
5961 vfree(memcg);
Hugh Dickins59927fb2012-03-15 15:17:07 -07005962}
Glauber Costa3afe36b2012-05-29 15:07:10 -07005963
Glauber Costac8b2a362012-12-18 14:22:13 -08005964
5965/*
5966 * Helpers for freeing a kmalloc()ed/vzalloc()ed mem_cgroup by RCU,
5967 * but in process context. The work_freeing structure is overlaid
5968 * on the rcu_freeing structure, which itself is overlaid on memsw.
5969 */
5970static void free_work(struct work_struct *work)
5971{
5972 struct mem_cgroup *memcg;
5973
5974 memcg = container_of(work, struct mem_cgroup, work_freeing);
5975 __mem_cgroup_free(memcg);
5976}
5977
Glauber Costa3afe36b2012-05-29 15:07:10 -07005978static void free_rcu(struct rcu_head *rcu_head)
Hugh Dickins59927fb2012-03-15 15:17:07 -07005979{
5980 struct mem_cgroup *memcg;
5981
5982 memcg = container_of(rcu_head, struct mem_cgroup, rcu_freeing);
Glauber Costa3afe36b2012-05-29 15:07:10 -07005983 INIT_WORK(&memcg->work_freeing, free_work);
Hugh Dickins59927fb2012-03-15 15:17:07 -07005984 schedule_work(&memcg->work_freeing);
5985}
5986
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005987static void mem_cgroup_get(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005988{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005989 atomic_inc(&memcg->refcnt);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005990}
5991
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005992static void __mem_cgroup_put(struct mem_cgroup *memcg, int count)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005993{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005994 if (atomic_sub_and_test(count, &memcg->refcnt)) {
5995 struct mem_cgroup *parent = parent_mem_cgroup(memcg);
Glauber Costac8b2a362012-12-18 14:22:13 -08005996 call_rcu(&memcg->rcu_freeing, free_rcu);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08005997 if (parent)
5998 mem_cgroup_put(parent);
5999 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08006000}
6001
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006002static void mem_cgroup_put(struct mem_cgroup *memcg)
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006003{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006004 __mem_cgroup_put(memcg, 1);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006005}
6006
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08006007/*
6008 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
6009 */
Glauber Costae1aab162011-12-11 21:47:03 +00006010struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08006011{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006012 if (!memcg->res.parent)
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08006013 return NULL;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006014 return mem_cgroup_from_res_counter(memcg->res.parent, res);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08006015}
Glauber Costae1aab162011-12-11 21:47:03 +00006016EXPORT_SYMBOL(parent_mem_cgroup);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07006017
Andrew Mortonc255a452012-07-31 16:43:02 -07006018#ifdef CONFIG_MEMCG_SWAP
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006019static void __init enable_swap_cgroup(void)
6020{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08006021 if (!mem_cgroup_disabled() && really_do_swap_account)
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006022 do_swap_account = 1;
6023}
6024#else
6025static void __init enable_swap_cgroup(void)
6026{
6027}
6028#endif
6029
Balbir Singhf64c3f52009-09-23 15:56:37 -07006030static int mem_cgroup_soft_limit_tree_init(void)
6031{
6032 struct mem_cgroup_tree_per_node *rtpn;
6033 struct mem_cgroup_tree_per_zone *rtpz;
6034 int tmp, node, zone;
6035
Bob Liu3ed28fa2012-01-12 17:19:04 -08006036 for_each_node(node) {
Balbir Singhf64c3f52009-09-23 15:56:37 -07006037 tmp = node;
6038 if (!node_state(node, N_NORMAL_MEMORY))
6039 tmp = -1;
6040 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
6041 if (!rtpn)
Michal Hockoc3cecc62012-01-12 17:18:50 -08006042 goto err_cleanup;
Balbir Singhf64c3f52009-09-23 15:56:37 -07006043
6044 soft_limit_tree.rb_tree_per_node[node] = rtpn;
6045
6046 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
6047 rtpz = &rtpn->rb_tree_per_zone[zone];
6048 rtpz->rb_root = RB_ROOT;
6049 spin_lock_init(&rtpz->lock);
6050 }
6051 }
6052 return 0;
Michal Hockoc3cecc62012-01-12 17:18:50 -08006053
6054err_cleanup:
Bob Liu3ed28fa2012-01-12 17:19:04 -08006055 for_each_node(node) {
Michal Hockoc3cecc62012-01-12 17:18:50 -08006056 if (!soft_limit_tree.rb_tree_per_node[node])
6057 break;
6058 kfree(soft_limit_tree.rb_tree_per_node[node]);
6059 soft_limit_tree.rb_tree_per_node[node] = NULL;
6060 }
6061 return 1;
6062
Balbir Singhf64c3f52009-09-23 15:56:37 -07006063}
6064
Li Zefan0eb253e2009-01-15 13:51:25 -08006065static struct cgroup_subsys_state * __ref
Tejun Heo92fb9742012-11-19 08:13:38 -08006066mem_cgroup_css_alloc(struct cgroup *cont)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006067{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006068 struct mem_cgroup *memcg, *parent;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07006069 long error = -ENOMEM;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006070 int node;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006071
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006072 memcg = mem_cgroup_alloc();
6073 if (!memcg)
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07006074 return ERR_PTR(error);
Pavel Emelianov78fb7462008-02-07 00:13:51 -08006075
Bob Liu3ed28fa2012-01-12 17:19:04 -08006076 for_each_node(node)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006077 if (alloc_mem_cgroup_per_zone_info(memcg, node))
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006078 goto free_out;
Balbir Singhf64c3f52009-09-23 15:56:37 -07006079
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006080 /* root ? */
Balbir Singh28dbc4b2009-01-07 18:08:05 -08006081 if (cont->parent == NULL) {
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08006082 int cpu;
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006083 enable_swap_cgroup();
Balbir Singh28dbc4b2009-01-07 18:08:05 -08006084 parent = NULL;
Balbir Singhf64c3f52009-09-23 15:56:37 -07006085 if (mem_cgroup_soft_limit_tree_init())
6086 goto free_out;
Hillf Dantona41c58a2011-12-19 17:11:57 -08006087 root_mem_cgroup = memcg;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08006088 for_each_possible_cpu(cpu) {
6089 struct memcg_stock_pcp *stock =
6090 &per_cpu(memcg_stock, cpu);
6091 INIT_WORK(&stock->work, drain_local_stock);
6092 }
Balbir Singh18f59ea2009-01-07 18:08:07 -08006093 } else {
Balbir Singh28dbc4b2009-01-07 18:08:05 -08006094 parent = mem_cgroup_from_cont(cont->parent);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006095 memcg->use_hierarchy = parent->use_hierarchy;
6096 memcg->oom_kill_disable = parent->oom_kill_disable;
Balbir Singh18f59ea2009-01-07 18:08:07 -08006097 }
Balbir Singh28dbc4b2009-01-07 18:08:05 -08006098
Balbir Singh18f59ea2009-01-07 18:08:07 -08006099 if (parent && parent->use_hierarchy) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006100 res_counter_init(&memcg->res, &parent->res);
6101 res_counter_init(&memcg->memsw, &parent->memsw);
Glauber Costa510fc4e2012-12-18 14:21:47 -08006102 res_counter_init(&memcg->kmem, &parent->kmem);
Glauber Costa55007d82012-12-18 14:22:38 -08006103
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08006104 /*
6105 * We increment refcnt of the parent to ensure that we can
6106 * safely access it on res_counter_charge/uncharge.
6107 * This refcnt will be decremented when freeing this
6108 * mem_cgroup(see mem_cgroup_put).
6109 */
6110 mem_cgroup_get(parent);
Balbir Singh18f59ea2009-01-07 18:08:07 -08006111 } else {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006112 res_counter_init(&memcg->res, NULL);
6113 res_counter_init(&memcg->memsw, NULL);
Glauber Costa510fc4e2012-12-18 14:21:47 -08006114 res_counter_init(&memcg->kmem, NULL);
Tejun Heo8c7f6ed2012-09-13 12:20:58 -07006115 /*
6116 * Deeper hierachy with use_hierarchy == false doesn't make
6117 * much sense so let cgroup subsystem know about this
6118 * unfortunate state in our controller.
6119 */
6120 if (parent && parent != root_mem_cgroup)
6121 mem_cgroup_subsys.broken_hierarchy = true;
Balbir Singh18f59ea2009-01-07 18:08:07 -08006122 }
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006123 memcg->last_scanned_node = MAX_NUMNODES;
6124 INIT_LIST_HEAD(&memcg->oom_notify);
Balbir Singh6d61ef42009-01-07 18:08:06 -08006125
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08006126 if (parent)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006127 memcg->swappiness = mem_cgroup_swappiness(parent);
6128 atomic_set(&memcg->refcnt, 1);
6129 memcg->move_charge_at_immigrate = 0;
6130 mutex_init(&memcg->thresholds_lock);
KAMEZAWA Hiroyuki312734c02012-03-21 16:34:24 -07006131 spin_lock_init(&memcg->move_lock);
Glauber Costacbe128e32012-04-09 19:36:34 -03006132
6133 error = memcg_init_kmem(memcg, &mem_cgroup_subsys);
6134 if (error) {
6135 /*
6136 * We call put now because our (and parent's) refcnts
6137 * are already in place. mem_cgroup_put() will internally
6138 * call __mem_cgroup_free, so return directly
6139 */
6140 mem_cgroup_put(memcg);
6141 return ERR_PTR(error);
6142 }
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006143 return &memcg->css;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006144free_out:
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006145 __mem_cgroup_free(memcg);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07006146 return ERR_PTR(error);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006147}
6148
Tejun Heo92fb9742012-11-19 08:13:38 -08006149static void mem_cgroup_css_offline(struct cgroup *cont)
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08006150{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006151 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07006152
Michal Hockoab5196c2012-10-26 13:37:32 +02006153 mem_cgroup_reparent_charges(memcg);
Glauber Costa1f458cb2012-12-18 14:22:50 -08006154 mem_cgroup_destroy_all_caches(memcg);
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08006155}
6156
Tejun Heo92fb9742012-11-19 08:13:38 -08006157static void mem_cgroup_css_free(struct cgroup *cont)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006158{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006159 struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
Daisuke Nishimurac268e992009-01-15 13:51:13 -08006160
Glauber Costa1d62e432012-04-09 19:36:33 -03006161 kmem_cgroup_destroy(memcg);
Glauber Costad1a4c0b2011-12-11 21:47:04 +00006162
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006163 mem_cgroup_put(memcg);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006164}
6165
Daisuke Nishimura02491442010-03-10 15:22:17 -08006166#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006167/* Handlers for move charge at task migration. */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006168#define PRECHARGE_COUNT_AT_ONCE 256
6169static int mem_cgroup_do_precharge(unsigned long count)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006170{
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006171 int ret = 0;
6172 int batch_count = PRECHARGE_COUNT_AT_ONCE;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006173 struct mem_cgroup *memcg = mc.to;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006174
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006175 if (mem_cgroup_is_root(memcg)) {
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006176 mc.precharge += count;
6177 /* we don't need css_get for root */
6178 return ret;
6179 }
6180 /* try to charge at once */
6181 if (count > 1) {
6182 struct res_counter *dummy;
6183 /*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006184 * "memcg" cannot be under rmdir() because we've already checked
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006185 * by cgroup_lock_live_cgroup() that it is not removed and we
6186 * are still under the same cgroup_mutex. So we can postpone
6187 * css_get().
6188 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006189 if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy))
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006190 goto one_by_one;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006191 if (do_swap_account && res_counter_charge(&memcg->memsw,
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006192 PAGE_SIZE * count, &dummy)) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006193 res_counter_uncharge(&memcg->res, PAGE_SIZE * count);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006194 goto one_by_one;
6195 }
6196 mc.precharge += count;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006197 return ret;
6198 }
6199one_by_one:
6200 /* fall back to one by one charge */
6201 while (count--) {
6202 if (signal_pending(current)) {
6203 ret = -EINTR;
6204 break;
6205 }
6206 if (!batch_count--) {
6207 batch_count = PRECHARGE_COUNT_AT_ONCE;
6208 cond_resched();
6209 }
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006210 ret = __mem_cgroup_try_charge(NULL,
6211 GFP_KERNEL, 1, &memcg, false);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08006212 if (ret)
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006213 /* mem_cgroup_clear_mc() will do uncharge later */
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08006214 return ret;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006215 mc.precharge++;
6216 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006217 return ret;
6218}
6219
6220/**
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006221 * get_mctgt_type - get target type of moving charge
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006222 * @vma: the vma the pte to be checked belongs
6223 * @addr: the address corresponding to the pte to be checked
6224 * @ptent: the pte to be checked
Daisuke Nishimura02491442010-03-10 15:22:17 -08006225 * @target: the pointer the target page or swap ent will be stored(can be NULL)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006226 *
6227 * Returns
6228 * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
6229 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
6230 * move charge. if @target is not NULL, the page is stored in target->page
6231 * with extra refcnt got(Callers should handle it).
Daisuke Nishimura02491442010-03-10 15:22:17 -08006232 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
6233 * target for charge migration. if @target is not NULL, the entry is stored
6234 * in target->ent.
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006235 *
6236 * Called with pte lock held.
6237 */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006238union mc_target {
6239 struct page *page;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006240 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006241};
6242
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006243enum mc_target_type {
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006244 MC_TARGET_NONE = 0,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006245 MC_TARGET_PAGE,
Daisuke Nishimura02491442010-03-10 15:22:17 -08006246 MC_TARGET_SWAP,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006247};
6248
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006249static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
6250 unsigned long addr, pte_t ptent)
6251{
6252 struct page *page = vm_normal_page(vma, addr, ptent);
6253
6254 if (!page || !page_mapped(page))
6255 return NULL;
6256 if (PageAnon(page)) {
6257 /* we don't move shared anon */
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07006258 if (!move_anon())
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006259 return NULL;
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006260 } else if (!move_file())
6261 /* we ignore mapcount for file pages */
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006262 return NULL;
6263 if (!get_page_unless_zero(page))
6264 return NULL;
6265
6266 return page;
6267}
6268
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07006269#ifdef CONFIG_SWAP
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006270static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
6271 unsigned long addr, pte_t ptent, swp_entry_t *entry)
6272{
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006273 struct page *page = NULL;
6274 swp_entry_t ent = pte_to_swp_entry(ptent);
6275
6276 if (!move_anon() || non_swap_entry(ent))
6277 return NULL;
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07006278 /*
6279 * Because lookup_swap_cache() updates some statistics counter,
6280 * we call find_get_page() with swapper_space directly.
6281 */
6282 page = find_get_page(&swapper_space, ent.val);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006283 if (do_swap_account)
6284 entry->val = ent.val;
6285
6286 return page;
6287}
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07006288#else
6289static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
6290 unsigned long addr, pte_t ptent, swp_entry_t *entry)
6291{
6292 return NULL;
6293}
6294#endif
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006295
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006296static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
6297 unsigned long addr, pte_t ptent, swp_entry_t *entry)
6298{
6299 struct page *page = NULL;
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006300 struct address_space *mapping;
6301 pgoff_t pgoff;
6302
6303 if (!vma->vm_file) /* anonymous vma */
6304 return NULL;
6305 if (!move_file())
6306 return NULL;
6307
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006308 mapping = vma->vm_file->f_mapping;
6309 if (pte_none(ptent))
6310 pgoff = linear_page_index(vma, addr);
6311 else /* pte_file(ptent) is true */
6312 pgoff = pte_to_pgoff(ptent);
6313
6314 /* page is moved even if it's not RSS of this task(page-faulted). */
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07006315 page = find_get_page(mapping, pgoff);
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006316
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07006317#ifdef CONFIG_SWAP
6318 /* shmem/tmpfs may report page out on swap: account for that too. */
6319 if (radix_tree_exceptional_entry(page)) {
6320 swp_entry_t swap = radix_to_swp_entry(page);
6321 if (do_swap_account)
6322 *entry = swap;
6323 page = find_get_page(&swapper_space, swap.val);
6324 }
6325#endif
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006326 return page;
6327}
6328
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006329static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006330 unsigned long addr, pte_t ptent, union mc_target *target)
6331{
Daisuke Nishimura02491442010-03-10 15:22:17 -08006332 struct page *page = NULL;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006333 struct page_cgroup *pc;
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006334 enum mc_target_type ret = MC_TARGET_NONE;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006335 swp_entry_t ent = { .val = 0 };
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006336
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006337 if (pte_present(ptent))
6338 page = mc_handle_present_pte(vma, addr, ptent);
6339 else if (is_swap_pte(ptent))
6340 page = mc_handle_swap_pte(vma, addr, ptent, &ent);
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006341 else if (pte_none(ptent) || pte_file(ptent))
6342 page = mc_handle_file_pte(vma, addr, ptent, &ent);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006343
6344 if (!page && !ent.val)
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006345 return ret;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006346 if (page) {
6347 pc = lookup_page_cgroup(page);
6348 /*
6349 * Do only loose check w/o page_cgroup lock.
6350 * mem_cgroup_move_account() checks the pc is valid or not under
6351 * the lock.
6352 */
6353 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
6354 ret = MC_TARGET_PAGE;
6355 if (target)
6356 target->page = page;
6357 }
6358 if (!ret || !target)
6359 put_page(page);
6360 }
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006361 /* There is a swap entry and a page doesn't exist or isn't charged */
6362 if (ent.val && !ret &&
Bob Liu9fb4b7c2012-01-12 17:18:48 -08006363 css_id(&mc.from->css) == lookup_swap_cgroup_id(ent)) {
KAMEZAWA Hiroyuki7f0f1542010-05-11 14:06:58 -07006364 ret = MC_TARGET_SWAP;
6365 if (target)
6366 target->ent = ent;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006367 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006368 return ret;
6369}
6370
Naoya Horiguchi12724852012-03-21 16:34:28 -07006371#ifdef CONFIG_TRANSPARENT_HUGEPAGE
6372/*
6373 * We don't consider swapping or file mapped pages because THP does not
6374 * support them for now.
6375 * Caller should make sure that pmd_trans_huge(pmd) is true.
6376 */
6377static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
6378 unsigned long addr, pmd_t pmd, union mc_target *target)
6379{
6380 struct page *page = NULL;
6381 struct page_cgroup *pc;
6382 enum mc_target_type ret = MC_TARGET_NONE;
6383
6384 page = pmd_page(pmd);
6385 VM_BUG_ON(!page || !PageHead(page));
6386 if (!move_anon())
6387 return ret;
6388 pc = lookup_page_cgroup(page);
6389 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
6390 ret = MC_TARGET_PAGE;
6391 if (target) {
6392 get_page(page);
6393 target->page = page;
6394 }
6395 }
6396 return ret;
6397}
6398#else
6399static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
6400 unsigned long addr, pmd_t pmd, union mc_target *target)
6401{
6402 return MC_TARGET_NONE;
6403}
6404#endif
6405
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006406static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
6407 unsigned long addr, unsigned long end,
6408 struct mm_walk *walk)
6409{
6410 struct vm_area_struct *vma = walk->private;
6411 pte_t *pte;
6412 spinlock_t *ptl;
6413
Naoya Horiguchi12724852012-03-21 16:34:28 -07006414 if (pmd_trans_huge_lock(pmd, vma) == 1) {
6415 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
6416 mc.precharge += HPAGE_PMD_NR;
6417 spin_unlock(&vma->vm_mm->page_table_lock);
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07006418 return 0;
Naoya Horiguchi12724852012-03-21 16:34:28 -07006419 }
Dave Hansen03319322011-03-22 16:32:56 -07006420
Andrea Arcangeli45f83ce2012-03-28 14:42:40 -07006421 if (pmd_trans_unstable(pmd))
6422 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006423 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
6424 for (; addr != end; pte++, addr += PAGE_SIZE)
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006425 if (get_mctgt_type(vma, addr, *pte, NULL))
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006426 mc.precharge++; /* increment precharge temporarily */
6427 pte_unmap_unlock(pte - 1, ptl);
6428 cond_resched();
6429
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006430 return 0;
6431}
6432
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006433static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
6434{
6435 unsigned long precharge;
6436 struct vm_area_struct *vma;
6437
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006438 down_read(&mm->mmap_sem);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006439 for (vma = mm->mmap; vma; vma = vma->vm_next) {
6440 struct mm_walk mem_cgroup_count_precharge_walk = {
6441 .pmd_entry = mem_cgroup_count_precharge_pte_range,
6442 .mm = mm,
6443 .private = vma,
6444 };
6445 if (is_vm_hugetlb_page(vma))
6446 continue;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006447 walk_page_range(vma->vm_start, vma->vm_end,
6448 &mem_cgroup_count_precharge_walk);
6449 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006450 up_read(&mm->mmap_sem);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006451
6452 precharge = mc.precharge;
6453 mc.precharge = 0;
6454
6455 return precharge;
6456}
6457
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006458static int mem_cgroup_precharge_mc(struct mm_struct *mm)
6459{
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006460 unsigned long precharge = mem_cgroup_count_precharge(mm);
6461
6462 VM_BUG_ON(mc.moving_task);
6463 mc.moving_task = current;
6464 return mem_cgroup_do_precharge(precharge);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006465}
6466
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006467/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
6468static void __mem_cgroup_clear_mc(void)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006469{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07006470 struct mem_cgroup *from = mc.from;
6471 struct mem_cgroup *to = mc.to;
6472
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006473 /* we must uncharge all the leftover precharges from mc.to */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006474 if (mc.precharge) {
6475 __mem_cgroup_cancel_charge(mc.to, mc.precharge);
6476 mc.precharge = 0;
6477 }
6478 /*
6479 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
6480 * we must uncharge here.
6481 */
6482 if (mc.moved_charge) {
6483 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
6484 mc.moved_charge = 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006485 }
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006486 /* we must fixup refcnts and charges */
6487 if (mc.moved_swap) {
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006488 /* uncharge swap account from the old cgroup */
6489 if (!mem_cgroup_is_root(mc.from))
6490 res_counter_uncharge(&mc.from->memsw,
6491 PAGE_SIZE * mc.moved_swap);
6492 __mem_cgroup_put(mc.from, mc.moved_swap);
6493
6494 if (!mem_cgroup_is_root(mc.to)) {
6495 /*
6496 * we charged both to->res and to->memsw, so we should
6497 * uncharge to->res.
6498 */
6499 res_counter_uncharge(&mc.to->res,
6500 PAGE_SIZE * mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006501 }
6502 /* we've already done mem_cgroup_get(mc.to) */
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006503 mc.moved_swap = 0;
6504 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006505 memcg_oom_recover(from);
6506 memcg_oom_recover(to);
6507 wake_up_all(&mc.waitq);
6508}
6509
6510static void mem_cgroup_clear_mc(void)
6511{
6512 struct mem_cgroup *from = mc.from;
6513
6514 /*
6515 * we must clear moving_task before waking up waiters at the end of
6516 * task migration.
6517 */
6518 mc.moving_task = NULL;
6519 __mem_cgroup_clear_mc();
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07006520 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006521 mc.from = NULL;
6522 mc.to = NULL;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07006523 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07006524 mem_cgroup_end_move(from);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006525}
6526
Li Zefan761b3ef2012-01-31 13:47:36 +08006527static int mem_cgroup_can_attach(struct cgroup *cgroup,
6528 struct cgroup_taskset *tset)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006529{
Tejun Heo2f7ee562011-12-12 18:12:21 -08006530 struct task_struct *p = cgroup_taskset_first(tset);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006531 int ret = 0;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006532 struct mem_cgroup *memcg = mem_cgroup_from_cont(cgroup);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006533
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006534 if (memcg->move_charge_at_immigrate) {
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006535 struct mm_struct *mm;
6536 struct mem_cgroup *from = mem_cgroup_from_task(p);
6537
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006538 VM_BUG_ON(from == memcg);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006539
6540 mm = get_task_mm(p);
6541 if (!mm)
6542 return 0;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006543 /* We move charges only when we move a owner of the mm */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006544 if (mm->owner == p) {
6545 VM_BUG_ON(mc.from);
6546 VM_BUG_ON(mc.to);
6547 VM_BUG_ON(mc.precharge);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006548 VM_BUG_ON(mc.moved_charge);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006549 VM_BUG_ON(mc.moved_swap);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07006550 mem_cgroup_start_move(from);
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07006551 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006552 mc.from = from;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006553 mc.to = memcg;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07006554 spin_unlock(&mc.lock);
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006555 /* We set mc.moving_task later */
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006556
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006557 ret = mem_cgroup_precharge_mc(mm);
6558 if (ret)
6559 mem_cgroup_clear_mc();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006560 }
6561 mmput(mm);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006562 }
6563 return ret;
6564}
6565
Li Zefan761b3ef2012-01-31 13:47:36 +08006566static void mem_cgroup_cancel_attach(struct cgroup *cgroup,
6567 struct cgroup_taskset *tset)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006568{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006569 mem_cgroup_clear_mc();
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006570}
6571
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006572static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
6573 unsigned long addr, unsigned long end,
6574 struct mm_walk *walk)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006575{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006576 int ret = 0;
6577 struct vm_area_struct *vma = walk->private;
6578 pte_t *pte;
6579 spinlock_t *ptl;
Naoya Horiguchi12724852012-03-21 16:34:28 -07006580 enum mc_target_type target_type;
6581 union mc_target target;
6582 struct page *page;
6583 struct page_cgroup *pc;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006584
Naoya Horiguchi12724852012-03-21 16:34:28 -07006585 /*
6586 * We don't take compound_lock() here but no race with splitting thp
6587 * happens because:
6588 * - if pmd_trans_huge_lock() returns 1, the relevant thp is not
6589 * under splitting, which means there's no concurrent thp split,
6590 * - if another thread runs into split_huge_page() just after we
6591 * entered this if-block, the thread must wait for page table lock
6592 * to be unlocked in __split_huge_page_splitting(), where the main
6593 * part of thp split is not executed yet.
6594 */
6595 if (pmd_trans_huge_lock(pmd, vma) == 1) {
Hugh Dickins62ade862012-05-18 11:28:34 -07006596 if (mc.precharge < HPAGE_PMD_NR) {
Naoya Horiguchi12724852012-03-21 16:34:28 -07006597 spin_unlock(&vma->vm_mm->page_table_lock);
6598 return 0;
6599 }
6600 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
6601 if (target_type == MC_TARGET_PAGE) {
6602 page = target.page;
6603 if (!isolate_lru_page(page)) {
6604 pc = lookup_page_cgroup(page);
6605 if (!mem_cgroup_move_account(page, HPAGE_PMD_NR,
KAMEZAWA Hiroyuki2f3479b2012-05-29 15:07:04 -07006606 pc, mc.from, mc.to)) {
Naoya Horiguchi12724852012-03-21 16:34:28 -07006607 mc.precharge -= HPAGE_PMD_NR;
6608 mc.moved_charge += HPAGE_PMD_NR;
6609 }
6610 putback_lru_page(page);
6611 }
6612 put_page(page);
6613 }
6614 spin_unlock(&vma->vm_mm->page_table_lock);
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07006615 return 0;
Naoya Horiguchi12724852012-03-21 16:34:28 -07006616 }
6617
Andrea Arcangeli45f83ce2012-03-28 14:42:40 -07006618 if (pmd_trans_unstable(pmd))
6619 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006620retry:
6621 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
6622 for (; addr != end; addr += PAGE_SIZE) {
6623 pte_t ptent = *(pte++);
Daisuke Nishimura02491442010-03-10 15:22:17 -08006624 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006625
6626 if (!mc.precharge)
6627 break;
6628
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006629 switch (get_mctgt_type(vma, addr, ptent, &target)) {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006630 case MC_TARGET_PAGE:
6631 page = target.page;
6632 if (isolate_lru_page(page))
6633 goto put;
6634 pc = lookup_page_cgroup(page);
Johannes Weiner7ec99d62011-03-23 16:42:36 -07006635 if (!mem_cgroup_move_account(page, 1, pc,
KAMEZAWA Hiroyuki2f3479b2012-05-29 15:07:04 -07006636 mc.from, mc.to)) {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006637 mc.precharge--;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006638 /* we uncharge from mc.from later. */
6639 mc.moved_charge++;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006640 }
6641 putback_lru_page(page);
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006642put: /* get_mctgt_type() gets the page */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006643 put_page(page);
6644 break;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006645 case MC_TARGET_SWAP:
6646 ent = target.ent;
Hugh Dickinse91cbb42012-05-29 15:06:51 -07006647 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08006648 mc.precharge--;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006649 /* we fixup refcnts and charges later. */
6650 mc.moved_swap++;
6651 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08006652 break;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006653 default:
6654 break;
6655 }
6656 }
6657 pte_unmap_unlock(pte - 1, ptl);
6658 cond_resched();
6659
6660 if (addr != end) {
6661 /*
6662 * We have consumed all precharges we got in can_attach().
6663 * We try charge one by one, but don't do any additional
6664 * charges to mc.to if we have failed in charge once in attach()
6665 * phase.
6666 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006667 ret = mem_cgroup_do_precharge(1);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006668 if (!ret)
6669 goto retry;
6670 }
6671
6672 return ret;
6673}
6674
6675static void mem_cgroup_move_charge(struct mm_struct *mm)
6676{
6677 struct vm_area_struct *vma;
6678
6679 lru_add_drain_all();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006680retry:
6681 if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
6682 /*
6683 * Someone who are holding the mmap_sem might be waiting in
6684 * waitq. So we cancel all extra charges, wake up all waiters,
6685 * and retry. Because we cancel precharges, we might not be able
6686 * to move enough charges, but moving charge is a best-effort
6687 * feature anyway, so it wouldn't be a big problem.
6688 */
6689 __mem_cgroup_clear_mc();
6690 cond_resched();
6691 goto retry;
6692 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006693 for (vma = mm->mmap; vma; vma = vma->vm_next) {
6694 int ret;
6695 struct mm_walk mem_cgroup_move_charge_walk = {
6696 .pmd_entry = mem_cgroup_move_charge_pte_range,
6697 .mm = mm,
6698 .private = vma,
6699 };
6700 if (is_vm_hugetlb_page(vma))
6701 continue;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006702 ret = walk_page_range(vma->vm_start, vma->vm_end,
6703 &mem_cgroup_move_charge_walk);
6704 if (ret)
6705 /*
6706 * means we have consumed all precharges and failed in
6707 * doing additional charge. Just abandon here.
6708 */
6709 break;
6710 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006711 up_read(&mm->mmap_sem);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006712}
6713
Li Zefan761b3ef2012-01-31 13:47:36 +08006714static void mem_cgroup_move_task(struct cgroup *cont,
6715 struct cgroup_taskset *tset)
Balbir Singh67e465a2008-02-07 00:13:54 -08006716{
Tejun Heo2f7ee562011-12-12 18:12:21 -08006717 struct task_struct *p = cgroup_taskset_first(tset);
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07006718 struct mm_struct *mm = get_task_mm(p);
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006719
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006720 if (mm) {
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07006721 if (mc.to)
6722 mem_cgroup_move_charge(mm);
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006723 mmput(mm);
6724 }
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07006725 if (mc.to)
6726 mem_cgroup_clear_mc();
Balbir Singh67e465a2008-02-07 00:13:54 -08006727}
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006728#else /* !CONFIG_MMU */
Li Zefan761b3ef2012-01-31 13:47:36 +08006729static int mem_cgroup_can_attach(struct cgroup *cgroup,
6730 struct cgroup_taskset *tset)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006731{
6732 return 0;
6733}
Li Zefan761b3ef2012-01-31 13:47:36 +08006734static void mem_cgroup_cancel_attach(struct cgroup *cgroup,
6735 struct cgroup_taskset *tset)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006736{
6737}
Li Zefan761b3ef2012-01-31 13:47:36 +08006738static void mem_cgroup_move_task(struct cgroup *cont,
6739 struct cgroup_taskset *tset)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006740{
6741}
6742#endif
Balbir Singh67e465a2008-02-07 00:13:54 -08006743
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006744struct cgroup_subsys mem_cgroup_subsys = {
6745 .name = "memory",
6746 .subsys_id = mem_cgroup_subsys_id,
Tejun Heo92fb9742012-11-19 08:13:38 -08006747 .css_alloc = mem_cgroup_css_alloc,
6748 .css_offline = mem_cgroup_css_offline,
6749 .css_free = mem_cgroup_css_free,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006750 .can_attach = mem_cgroup_can_attach,
6751 .cancel_attach = mem_cgroup_cancel_attach,
Balbir Singh67e465a2008-02-07 00:13:54 -08006752 .attach = mem_cgroup_move_task,
Tejun Heo6bc10342012-04-01 12:09:55 -07006753 .base_cftypes = mem_cgroup_files,
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006754 .early_init = 0,
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07006755 .use_id = 1,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006756};
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006757
Tejun Heo154b4542012-12-20 15:05:40 -08006758/*
6759 * The rest of init is performed during ->css_alloc() for root css which
6760 * happens before initcalls. hotcpu_notifier() can't be done together as
6761 * it would introduce circular locking by adding cgroup_lock -> cpu hotplug
6762 * dependency. Do it from a subsys_initcall().
6763 */
6764static int __init mem_cgroup_init(void)
6765{
6766 hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
6767 return 0;
6768}
6769subsys_initcall(mem_cgroup_init);
6770
Andrew Mortonc255a452012-07-31 16:43:02 -07006771#ifdef CONFIG_MEMCG_SWAP
Michal Hockoa42c3902010-11-24 12:57:08 -08006772static int __init enable_swap_account(char *s)
6773{
6774 /* consider enabled if no parameter or 1 is given */
Michal Hockoa2c89902011-05-24 17:12:50 -07006775 if (!strcmp(s, "1"))
Michal Hockoa42c3902010-11-24 12:57:08 -08006776 really_do_swap_account = 1;
Michal Hockoa2c89902011-05-24 17:12:50 -07006777 else if (!strcmp(s, "0"))
Michal Hockoa42c3902010-11-24 12:57:08 -08006778 really_do_swap_account = 0;
6779 return 1;
6780}
Michal Hockoa2c89902011-05-24 17:12:50 -07006781__setup("swapaccount=", enable_swap_account);
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006782
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006783#endif