blob: 3427de9897a59b63a66ccc916ef4d87759cfbbfb [file] [log] [blame]
Balbir Singh8cdea7c2008-02-07 00:13:50 -08001/* memcontrol.c - Memory Controller
2 *
3 * Copyright IBM Corporation, 2007
4 * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5 *
Pavel Emelianov78fb7462008-02-07 00:13:51 -08006 * Copyright 2007 OpenVZ SWsoft Inc
7 * Author: Pavel Emelianov <xemul@openvz.org>
8 *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08009 * Memory thresholds
10 * Copyright (C) 2009 Nokia Corporation
11 * Author: Kirill A. Shutemov
12 *
Glauber Costa7ae1e1d2012-12-18 14:21:56 -080013 * Kernel Memory Controller
14 * Copyright (C) 2012 Parallels Inc. and Google Inc.
15 * Authors: Glauber Costa and Suleiman Souhlal
16 *
Balbir Singh8cdea7c2008-02-07 00:13:50 -080017 * This program is free software; you can redistribute it and/or modify
18 * it under the terms of the GNU General Public License as published by
19 * the Free Software Foundation; either version 2 of the License, or
20 * (at your option) any later version.
21 *
22 * This program is distributed in the hope that it will be useful,
23 * but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU General Public License for more details.
26 */
27
28#include <linux/res_counter.h>
29#include <linux/memcontrol.h>
30#include <linux/cgroup.h>
Pavel Emelianov78fb7462008-02-07 00:13:51 -080031#include <linux/mm.h>
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -080032#include <linux/hugetlb.h>
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -080033#include <linux/pagemap.h>
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -080034#include <linux/smp.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080035#include <linux/page-flags.h>
Balbir Singh66e17072008-02-07 00:13:56 -080036#include <linux/backing-dev.h>
Balbir Singh8a9f3cc2008-02-07 00:13:53 -080037#include <linux/bit_spinlock.h>
38#include <linux/rcupdate.h>
Balbir Singhe2224322009-04-02 16:57:39 -070039#include <linux/limits.h>
Paul Gortmakerb9e15ba2011-05-26 16:00:52 -040040#include <linux/export.h>
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -080041#include <linux/mutex.h>
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -070042#include <linux/rbtree.h>
Balbir Singhb6ac57d2008-04-29 01:00:19 -070043#include <linux/slab.h>
Balbir Singh66e17072008-02-07 00:13:56 -080044#include <linux/swap.h>
Daisuke Nishimura02491442010-03-10 15:22:17 -080045#include <linux/swapops.h>
Balbir Singh66e17072008-02-07 00:13:56 -080046#include <linux/spinlock.h>
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -080047#include <linux/eventfd.h>
48#include <linux/sort.h>
Balbir Singh66e17072008-02-07 00:13:56 -080049#include <linux/fs.h>
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -080050#include <linux/seq_file.h>
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -070051#include <linux/vmalloc.h>
Anton Vorontsov70ddf632013-04-29 15:08:31 -070052#include <linux/vmpressure.h>
Christoph Lameterb69408e2008-10-18 20:26:14 -070053#include <linux/mm_inline.h>
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -070054#include <linux/page_cgroup.h>
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -080055#include <linux/cpu.h>
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -070056#include <linux/oom.h>
Johannes Weiner0056f4e2013-10-31 16:34:14 -070057#include <linux/lockdep.h>
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -080058#include "internal.h"
Glauber Costad1a4c0b2011-12-11 21:47:04 +000059#include <net/sock.h>
Michal Hocko4bd2c1e2012-10-08 16:33:10 -070060#include <net/ip.h>
Glauber Costad1a4c0b2011-12-11 21:47:04 +000061#include <net/tcp_memcontrol.h>
Balbir Singh8cdea7c2008-02-07 00:13:50 -080062
Balbir Singh8697d332008-02-07 00:13:59 -080063#include <asm/uaccess.h>
64
KOSAKI Motohirocc8e9702010-08-09 17:19:57 -070065#include <trace/events/vmscan.h>
66
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070067struct cgroup_subsys mem_cgroup_subsys __read_mostly;
David Rientjes68ae5642012-12-12 13:51:57 -080068EXPORT_SYMBOL(mem_cgroup_subsys);
69
KAMEZAWA Hiroyukia181b0e2008-07-25 01:47:08 -070070#define MEM_CGROUP_RECLAIM_RETRIES 5
Kirill A. Shutemov6bbda352012-05-29 15:06:55 -070071static struct mem_cgroup *root_mem_cgroup __read_mostly;
Balbir Singh8cdea7c2008-02-07 00:13:50 -080072
Andrew Mortonc255a452012-07-31 16:43:02 -070073#ifdef CONFIG_MEMCG_SWAP
Li Zefan338c8432009-06-17 16:27:15 -070074/* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080075int do_swap_account __read_mostly;
Michal Hockoa42c3902010-11-24 12:57:08 -080076
77/* for remember boot option*/
Andrew Mortonc255a452012-07-31 16:43:02 -070078#ifdef CONFIG_MEMCG_SWAP_ENABLED
Michal Hockoa42c3902010-11-24 12:57:08 -080079static int really_do_swap_account __initdata = 1;
80#else
81static int really_do_swap_account __initdata = 0;
82#endif
83
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080084#else
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -070085#define do_swap_account 0
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -080086#endif
87
88
Johannes Weineraf7c4b02012-05-29 15:07:08 -070089static const char * const mem_cgroup_stat_names[] = {
90 "cache",
91 "rss",
David Rientjesb070e652013-05-07 16:18:09 -070092 "rss_huge",
Johannes Weineraf7c4b02012-05-29 15:07:08 -070093 "mapped_file",
Sha Zhengju3ea67d02013-09-12 15:13:53 -070094 "writeback",
Johannes Weineraf7c4b02012-05-29 15:07:08 -070095 "swap",
96};
97
Johannes Weinere9f89742011-03-23 16:42:37 -070098enum mem_cgroup_events_index {
99 MEM_CGROUP_EVENTS_PGPGIN, /* # of pages paged in */
100 MEM_CGROUP_EVENTS_PGPGOUT, /* # of pages paged out */
Ying Han456f9982011-05-26 16:25:38 -0700101 MEM_CGROUP_EVENTS_PGFAULT, /* # of page-faults */
102 MEM_CGROUP_EVENTS_PGMAJFAULT, /* # of major page-faults */
Johannes Weinere9f89742011-03-23 16:42:37 -0700103 MEM_CGROUP_EVENTS_NSTATS,
104};
Johannes Weineraf7c4b02012-05-29 15:07:08 -0700105
106static const char * const mem_cgroup_events_names[] = {
107 "pgpgin",
108 "pgpgout",
109 "pgfault",
110 "pgmajfault",
111};
112
Sha Zhengju58cf1882013-02-22 16:32:05 -0800113static const char * const mem_cgroup_lru_names[] = {
114 "inactive_anon",
115 "active_anon",
116 "inactive_file",
117 "active_file",
118 "unevictable",
119};
120
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700121/*
122 * Per memcg event counter is incremented at every pagein/pageout. With THP,
123 * it will be incremated by the number of pages. This counter is used for
124 * for trigger some periodic events. This is straightforward and better
125 * than using jiffies etc. to handle periodic memcg event.
126 */
127enum mem_cgroup_events_target {
128 MEM_CGROUP_TARGET_THRESH,
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -0700129 MEM_CGROUP_TARGET_SOFTLIMIT,
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -0700130 MEM_CGROUP_TARGET_NUMAINFO,
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700131 MEM_CGROUP_NTARGETS,
132};
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700133#define THRESHOLDS_EVENTS_TARGET 128
134#define SOFTLIMIT_EVENTS_TARGET 1024
135#define NUMAINFO_EVENTS_TARGET 1024
Johannes Weinere9f89742011-03-23 16:42:37 -0700136
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800137struct mem_cgroup_stat_cpu {
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700138 long count[MEM_CGROUP_STAT_NSTATS];
Johannes Weinere9f89742011-03-23 16:42:37 -0700139 unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
Johannes Weiner13114712012-05-29 15:07:07 -0700140 unsigned long nr_page_events;
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700141 unsigned long targets[MEM_CGROUP_NTARGETS];
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800142};
143
Johannes Weiner527a5ec2012-01-12 17:17:55 -0800144struct mem_cgroup_reclaim_iter {
Michal Hocko5f578162013-04-29 15:07:17 -0700145 /*
146 * last scanned hierarchy member. Valid only if last_dead_count
147 * matches memcg->dead_count of the hierarchy root group.
148 */
Michal Hocko542f85f2013-04-29 15:07:15 -0700149 struct mem_cgroup *last_visited;
Michal Hocko5f578162013-04-29 15:07:17 -0700150 unsigned long last_dead_count;
151
Johannes Weiner527a5ec2012-01-12 17:17:55 -0800152 /* scan generation, increased every round-trip */
153 unsigned int generation;
154};
155
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800156/*
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800157 * per-zone information in memory controller.
158 */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800159struct mem_cgroup_per_zone {
Johannes Weiner6290df52012-01-12 17:18:10 -0800160 struct lruvec lruvec;
Hugh Dickins1eb49272012-03-21 16:34:19 -0700161 unsigned long lru_size[NR_LRU_LISTS];
KOSAKI Motohiro3e2f41f2009-01-07 18:08:20 -0800162
Johannes Weiner527a5ec2012-01-12 17:17:55 -0800163 struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];
164
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -0700165 struct rb_node tree_node; /* RB tree node */
166 unsigned long long usage_in_excess;/* Set to the value by which */
167 /* the soft limit is exceeded*/
168 bool on_tree;
Hugh Dickinsd79154b2012-03-21 16:34:18 -0700169 struct mem_cgroup *memcg; /* Back pointer, we cannot */
Balbir Singh4e416952009-09-23 15:56:39 -0700170 /* use container_of */
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800171};
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800172
173struct mem_cgroup_per_node {
174 struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
175};
176
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -0700177/*
178 * Cgroups above their limits are maintained in a RB-Tree, independent of
179 * their hierarchy representation
180 */
181
182struct mem_cgroup_tree_per_zone {
183 struct rb_root rb_root;
184 spinlock_t lock;
185};
186
187struct mem_cgroup_tree_per_node {
188 struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
189};
190
191struct mem_cgroup_tree {
192 struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
193};
194
195static struct mem_cgroup_tree soft_limit_tree __read_mostly;
196
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800197struct mem_cgroup_threshold {
198 struct eventfd_ctx *eventfd;
199 u64 threshold;
200};
201
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700202/* For threshold */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800203struct mem_cgroup_threshold_ary {
Sha Zhengju748dad32012-05-29 15:06:57 -0700204 /* An array index points to threshold just below or equal to usage. */
Phil Carmody5407a562010-05-26 14:42:42 -0700205 int current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800206 /* Size of entries[] */
207 unsigned int size;
208 /* Array of thresholds */
209 struct mem_cgroup_threshold entries[0];
210};
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700211
212struct mem_cgroup_thresholds {
213 /* Primary thresholds array */
214 struct mem_cgroup_threshold_ary *primary;
215 /*
216 * Spare threshold array.
217 * This is needed to make mem_cgroup_unregister_event() "never fail".
218 * It must be able to store at least primary->size - 1 entries.
219 */
220 struct mem_cgroup_threshold_ary *spare;
221};
222
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700223/* for OOM */
224struct mem_cgroup_eventfd_list {
225 struct list_head list;
226 struct eventfd_ctx *eventfd;
227};
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800228
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700229static void mem_cgroup_threshold(struct mem_cgroup *memcg);
230static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800231
Balbir Singhf64c3f52009-09-23 15:56:37 -0700232/*
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800233 * The memory controller data structure. The memory controller controls both
234 * page cache and RSS per cgroup. We would eventually like to provide
235 * statistics based on the statistics developed by Rik Van Riel for clock-pro,
236 * to help the administrator determine what knobs to tune.
237 *
238 * TODO: Add a water mark for the memory controller. Reclaim will begin when
Balbir Singh8a9f3cc2008-02-07 00:13:53 -0800239 * we hit the water mark. May be even add a low water mark, such that
240 * no reclaim occurs from a cgroup at it's low water mark, this is
241 * a feature that will be implemented much later in the future.
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800242 */
243struct mem_cgroup {
244 struct cgroup_subsys_state css;
245 /*
246 * the counter to account for memory usage
247 */
248 struct res_counter res;
Hugh Dickins59927fb2012-03-15 15:17:07 -0700249
Anton Vorontsov70ddf632013-04-29 15:08:31 -0700250 /* vmpressure notifications */
251 struct vmpressure vmpressure;
252
Li Zefan465939a2013-07-08 16:00:38 -0700253 /*
254 * the counter to account for mem+swap usage.
255 */
256 struct res_counter memsw;
Hugh Dickins59927fb2012-03-15 15:17:07 -0700257
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800258 /*
Glauber Costa510fc4e2012-12-18 14:21:47 -0800259 * the counter to account for kernel memory usage.
260 */
261 struct res_counter kmem;
262 /*
Balbir Singh18f59ea2009-01-07 18:08:07 -0800263 * Should the accounting and control be hierarchical, per subtree?
264 */
265 bool use_hierarchy;
Glauber Costa510fc4e2012-12-18 14:21:47 -0800266 unsigned long kmem_account_flags; /* See KMEM_ACCOUNTED_*, below */
Michal Hocko79dfdac2011-07-26 16:08:23 -0700267
268 bool oom_lock;
269 atomic_t under_oom;
Johannes Weiner3812c8c2013-09-12 15:13:44 -0700270 atomic_t oom_wakeups;
Michal Hocko79dfdac2011-07-26 16:08:23 -0700271
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -0700272 int swappiness;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -0700273 /* OOM-Killer disable */
274 int oom_kill_disable;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -0800275
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -0700276 /* set when res.limit == memsw.limit */
277 bool memsw_is_minimum;
278
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800279 /* protect arrays of thresholds */
280 struct mutex thresholds_lock;
281
282 /* thresholds for memory usage. RCU-protected */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700283 struct mem_cgroup_thresholds thresholds;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -0700284
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800285 /* thresholds for mem+swap usage. RCU-protected */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -0700286 struct mem_cgroup_thresholds memsw_thresholds;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -0700287
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700288 /* For oom notifier event fd */
289 struct list_head oom_notify;
Johannes Weiner185efc02011-09-14 16:21:58 -0700290
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800291 /*
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800292 * Should we move charges of a task when a task is moved into this
293 * mem_cgroup ? And what type of charges should we move ?
294 */
Andrew Mortonf894ffa2013-09-12 15:13:35 -0700295 unsigned long move_charge_at_immigrate;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800296 /*
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -0700297 * set > 0 if pages under this cgroup are moving to other cgroup.
298 */
299 atomic_t moving_account;
KAMEZAWA Hiroyuki312734c02012-03-21 16:34:24 -0700300 /* taken only while moving_account > 0 */
301 spinlock_t move_lock;
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -0700302 /*
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800303 * percpu counter.
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800304 */
Kirill A. Shutemov3a7951b2012-05-29 15:06:56 -0700305 struct mem_cgroup_stat_cpu __percpu *stat;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700306 /*
307 * used when a cpu is offlined or other synchronizations
308 * See mem_cgroup_read_stat().
309 */
310 struct mem_cgroup_stat_cpu nocpu_base;
311 spinlock_t pcp_counter_lock;
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000312
Michal Hocko5f578162013-04-29 15:07:17 -0700313 atomic_t dead_count;
Michal Hocko4bd2c1e2012-10-08 16:33:10 -0700314#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_INET)
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000315 struct tcp_memcontrol tcp_mem;
316#endif
Glauber Costa2633d7a2012-12-18 14:22:34 -0800317#if defined(CONFIG_MEMCG_KMEM)
318 /* analogous to slab_common's slab_caches list. per-memcg */
319 struct list_head memcg_slab_caches;
320 /* Not a spinlock, we can take a lot of time walking the list */
321 struct mutex slab_caches_mutex;
322 /* Index in the kmem_cache->memcg_params->memcg_caches array */
323 int kmemcg_id;
324#endif
Glauber Costa45cf7eb2013-02-22 16:34:49 -0800325
326 int last_scanned_node;
327#if MAX_NUMNODES > 1
328 nodemask_t scan_nodes;
329 atomic_t numainfo_events;
330 atomic_t numainfo_updating;
331#endif
Anton Vorontsov70ddf632013-04-29 15:08:31 -0700332
Johannes Weiner54f72fe2013-07-08 15:59:49 -0700333 struct mem_cgroup_per_node *nodeinfo[0];
334 /* WARNING: nodeinfo must be the last member here */
Balbir Singh8cdea7c2008-02-07 00:13:50 -0800335};
336
Glauber Costa45cf7eb2013-02-22 16:34:49 -0800337static size_t memcg_size(void)
338{
339 return sizeof(struct mem_cgroup) +
340 nr_node_ids * sizeof(struct mem_cgroup_per_node);
341}
342
Glauber Costa510fc4e2012-12-18 14:21:47 -0800343/* internal only representation about the status of kmem accounting. */
344enum {
345 KMEM_ACCOUNTED_ACTIVE = 0, /* accounted by this cgroup itself */
Glauber Costaa8964b92012-12-18 14:22:09 -0800346 KMEM_ACCOUNTED_ACTIVATED, /* static key enabled. */
Glauber Costa7de37682012-12-18 14:22:07 -0800347 KMEM_ACCOUNTED_DEAD, /* dead memcg with pending kmem charges */
Glauber Costa510fc4e2012-12-18 14:21:47 -0800348};
349
Glauber Costaa8964b92012-12-18 14:22:09 -0800350/* We account when limit is on, but only after call sites are patched */
351#define KMEM_ACCOUNTED_MASK \
352 ((1 << KMEM_ACCOUNTED_ACTIVE) | (1 << KMEM_ACCOUNTED_ACTIVATED))
Glauber Costa510fc4e2012-12-18 14:21:47 -0800353
354#ifdef CONFIG_MEMCG_KMEM
355static inline void memcg_kmem_set_active(struct mem_cgroup *memcg)
356{
357 set_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
358}
Glauber Costa7de37682012-12-18 14:22:07 -0800359
360static bool memcg_kmem_is_active(struct mem_cgroup *memcg)
361{
362 return test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags);
363}
364
Glauber Costaa8964b92012-12-18 14:22:09 -0800365static void memcg_kmem_set_activated(struct mem_cgroup *memcg)
366{
367 set_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
368}
369
Glauber Costa55007d82012-12-18 14:22:38 -0800370static void memcg_kmem_clear_activated(struct mem_cgroup *memcg)
371{
372 clear_bit(KMEM_ACCOUNTED_ACTIVATED, &memcg->kmem_account_flags);
373}
374
Glauber Costa7de37682012-12-18 14:22:07 -0800375static void memcg_kmem_mark_dead(struct mem_cgroup *memcg)
376{
Li Zefan10d5ebf2013-07-08 16:00:33 -0700377 /*
378 * Our caller must use css_get() first, because memcg_uncharge_kmem()
379 * will call css_put() if it sees the memcg is dead.
380 */
381 smp_wmb();
Glauber Costa7de37682012-12-18 14:22:07 -0800382 if (test_bit(KMEM_ACCOUNTED_ACTIVE, &memcg->kmem_account_flags))
383 set_bit(KMEM_ACCOUNTED_DEAD, &memcg->kmem_account_flags);
384}
385
386static bool memcg_kmem_test_and_clear_dead(struct mem_cgroup *memcg)
387{
388 return test_and_clear_bit(KMEM_ACCOUNTED_DEAD,
389 &memcg->kmem_account_flags);
390}
Glauber Costa510fc4e2012-12-18 14:21:47 -0800391#endif
392
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800393/* Stuffs for move charges at task migration. */
394/*
Glauber Costaee5e8472013-02-22 16:34:50 -0800395 * Types of charges to be moved. "move_charge_at_immitgrate" and
396 * "immigrate_flags" are treated as a left-shifted bitmap of these types.
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800397 */
398enum move_type {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800399 MOVE_CHARGE_TYPE_ANON, /* private anonymous page and swap of it */
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700400 MOVE_CHARGE_TYPE_FILE, /* file page(including tmpfs) and swap of it */
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -0800401 NR_MOVE_TYPE,
402};
403
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800404/* "mc" and its members are protected by cgroup_mutex */
405static struct move_charge_struct {
Daisuke Nishimurab1dd6932010-11-24 12:57:06 -0800406 spinlock_t lock; /* for from, to */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800407 struct mem_cgroup *from;
408 struct mem_cgroup *to;
Glauber Costaee5e8472013-02-22 16:34:50 -0800409 unsigned long immigrate_flags;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800410 unsigned long precharge;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -0800411 unsigned long moved_charge;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -0800412 unsigned long moved_swap;
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800413 struct task_struct *moving_task; /* a task moving charges */
414 wait_queue_head_t waitq; /* a waitq for other context */
415} mc = {
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -0700416 .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
Daisuke Nishimura8033b972010-03-10 15:22:16 -0800417 .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
418};
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -0800419
Daisuke Nishimura90254a62010-05-26 14:42:38 -0700420static bool move_anon(void)
421{
Glauber Costaee5e8472013-02-22 16:34:50 -0800422 return test_bit(MOVE_CHARGE_TYPE_ANON, &mc.immigrate_flags);
Daisuke Nishimura90254a62010-05-26 14:42:38 -0700423}
424
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700425static bool move_file(void)
426{
Glauber Costaee5e8472013-02-22 16:34:50 -0800427 return test_bit(MOVE_CHARGE_TYPE_FILE, &mc.immigrate_flags);
Daisuke Nishimura87946a72010-05-26 14:42:39 -0700428}
429
Balbir Singh4e416952009-09-23 15:56:39 -0700430/*
431 * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
432 * limit reclaim to prevent infinite loops, if they ever occur.
433 */
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700434#define MEM_CGROUP_MAX_RECLAIM_LOOPS 100
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -0700435#define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS 2
Balbir Singh4e416952009-09-23 15:56:39 -0700436
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -0800437enum charge_type {
438 MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -0700439 MEM_CGROUP_CHARGE_TYPE_ANON,
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -0800440 MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -0700441 MEM_CGROUP_CHARGE_TYPE_DROP, /* a page was unused swap cache */
KAMEZAWA Hiroyukic05555b2008-10-18 20:28:11 -0700442 NR_CHARGE_TYPE,
443};
444
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800445/* for encoding cft->private value on file */
Glauber Costa86ae53e2012-12-18 14:21:45 -0800446enum res_type {
447 _MEM,
448 _MEMSWAP,
449 _OOM_TYPE,
Glauber Costa510fc4e2012-12-18 14:21:47 -0800450 _KMEM,
Glauber Costa86ae53e2012-12-18 14:21:45 -0800451};
452
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -0700453#define MEMFILE_PRIVATE(x, val) ((x) << 16 | (val))
454#define MEMFILE_TYPE(val) ((val) >> 16 & 0xffff)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800455#define MEMFILE_ATTR(val) ((val) & 0xffff)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -0700456/* Used for OOM nofiier */
457#define OOM_CONTROL (0)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800458
Balbir Singh75822b42009-09-23 15:56:38 -0700459/*
460 * Reclaim flags for mem_cgroup_hierarchical_reclaim
461 */
462#define MEM_CGROUP_RECLAIM_NOSWAP_BIT 0x0
463#define MEM_CGROUP_RECLAIM_NOSWAP (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
464#define MEM_CGROUP_RECLAIM_SHRINK_BIT 0x1
465#define MEM_CGROUP_RECLAIM_SHRINK (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
466
Glauber Costa09998212013-02-22 16:34:55 -0800467/*
468 * The memcg_create_mutex will be held whenever a new cgroup is created.
469 * As a consequence, any change that needs to protect against new child cgroups
470 * appearing has to hold it as well.
471 */
472static DEFINE_MUTEX(memcg_create_mutex);
473
Wanpeng Lib2145142012-07-31 16:46:01 -0700474struct mem_cgroup *mem_cgroup_from_css(struct cgroup_subsys_state *s)
475{
Tejun Heoa7c6d552013-08-08 20:11:23 -0400476 return s ? container_of(s, struct mem_cgroup, css) : NULL;
Wanpeng Lib2145142012-07-31 16:46:01 -0700477}
478
Anton Vorontsov70ddf632013-04-29 15:08:31 -0700479/* Some nice accessors for the vmpressure. */
480struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
481{
482 if (!memcg)
483 memcg = root_mem_cgroup;
484 return &memcg->vmpressure;
485}
486
487struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
488{
489 return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
490}
491
492struct vmpressure *css_to_vmpressure(struct cgroup_subsys_state *css)
493{
494 return &mem_cgroup_from_css(css)->vmpressure;
495}
496
Michal Hocko7ffc0ed2012-10-08 16:33:13 -0700497static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
498{
499 return (memcg == root_mem_cgroup);
500}
501
Glauber Costae1aab162011-12-11 21:47:03 +0000502/* Writing them here to avoid exposing memcg's inner layout */
Michal Hocko4bd2c1e2012-10-08 16:33:10 -0700503#if defined(CONFIG_INET) && defined(CONFIG_MEMCG_KMEM)
Glauber Costae1aab162011-12-11 21:47:03 +0000504
Glauber Costae1aab162011-12-11 21:47:03 +0000505void sock_update_memcg(struct sock *sk)
506{
Glauber Costa376be5f2012-01-20 04:57:14 +0000507 if (mem_cgroup_sockets_enabled) {
Glauber Costae1aab162011-12-11 21:47:03 +0000508 struct mem_cgroup *memcg;
Glauber Costa3f134612012-05-29 15:07:11 -0700509 struct cg_proto *cg_proto;
Glauber Costae1aab162011-12-11 21:47:03 +0000510
511 BUG_ON(!sk->sk_prot->proto_cgroup);
512
Glauber Costaf3f511e2012-01-05 20:16:39 +0000513 /* Socket cloning can throw us here with sk_cgrp already
514 * filled. It won't however, necessarily happen from
515 * process context. So the test for root memcg given
516 * the current task's memcg won't help us in this case.
517 *
518 * Respecting the original socket's memcg is a better
519 * decision in this case.
520 */
521 if (sk->sk_cgrp) {
522 BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
Li Zefan5347e5a2013-07-08 16:00:30 -0700523 css_get(&sk->sk_cgrp->memcg->css);
Glauber Costaf3f511e2012-01-05 20:16:39 +0000524 return;
525 }
526
Glauber Costae1aab162011-12-11 21:47:03 +0000527 rcu_read_lock();
528 memcg = mem_cgroup_from_task(current);
Glauber Costa3f134612012-05-29 15:07:11 -0700529 cg_proto = sk->sk_prot->proto_cgroup(memcg);
Li Zefan5347e5a2013-07-08 16:00:30 -0700530 if (!mem_cgroup_is_root(memcg) &&
531 memcg_proto_active(cg_proto) && css_tryget(&memcg->css)) {
Glauber Costa3f134612012-05-29 15:07:11 -0700532 sk->sk_cgrp = cg_proto;
Glauber Costae1aab162011-12-11 21:47:03 +0000533 }
534 rcu_read_unlock();
535 }
536}
537EXPORT_SYMBOL(sock_update_memcg);
538
539void sock_release_memcg(struct sock *sk)
540{
Glauber Costa376be5f2012-01-20 04:57:14 +0000541 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
Glauber Costae1aab162011-12-11 21:47:03 +0000542 struct mem_cgroup *memcg;
543 WARN_ON(!sk->sk_cgrp->memcg);
544 memcg = sk->sk_cgrp->memcg;
Li Zefan5347e5a2013-07-08 16:00:30 -0700545 css_put(&sk->sk_cgrp->memcg->css);
Glauber Costae1aab162011-12-11 21:47:03 +0000546 }
547}
Glauber Costad1a4c0b2011-12-11 21:47:04 +0000548
549struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
550{
551 if (!memcg || mem_cgroup_is_root(memcg))
552 return NULL;
553
554 return &memcg->tcp_mem.cg_proto;
555}
556EXPORT_SYMBOL(tcp_proto_cgroup);
Glauber Costae1aab162011-12-11 21:47:03 +0000557
Glauber Costa3f134612012-05-29 15:07:11 -0700558static void disarm_sock_keys(struct mem_cgroup *memcg)
559{
560 if (!memcg_proto_activated(&memcg->tcp_mem.cg_proto))
561 return;
562 static_key_slow_dec(&memcg_socket_limit_enabled);
563}
564#else
565static void disarm_sock_keys(struct mem_cgroup *memcg)
566{
567}
568#endif
569
Glauber Costaa8964b92012-12-18 14:22:09 -0800570#ifdef CONFIG_MEMCG_KMEM
Glauber Costa55007d82012-12-18 14:22:38 -0800571/*
572 * This will be the memcg's index in each cache's ->memcg_params->memcg_caches.
573 * There are two main reasons for not using the css_id for this:
574 * 1) this works better in sparse environments, where we have a lot of memcgs,
575 * but only a few kmem-limited. Or also, if we have, for instance, 200
576 * memcgs, and none but the 200th is kmem-limited, we'd have to have a
577 * 200 entry array for that.
578 *
579 * 2) In order not to violate the cgroup API, we would like to do all memory
580 * allocation in ->create(). At that point, we haven't yet allocated the
581 * css_id. Having a separate index prevents us from messing with the cgroup
582 * core for this
583 *
584 * The current size of the caches array is stored in
585 * memcg_limited_groups_array_size. It will double each time we have to
586 * increase it.
587 */
588static DEFINE_IDA(kmem_limited_groups);
Glauber Costa749c5412012-12-18 14:23:01 -0800589int memcg_limited_groups_array_size;
590
Glauber Costa55007d82012-12-18 14:22:38 -0800591/*
592 * MIN_SIZE is different than 1, because we would like to avoid going through
593 * the alloc/free process all the time. In a small machine, 4 kmem-limited
594 * cgroups is a reasonable guess. In the future, it could be a parameter or
595 * tunable, but that is strictly not necessary.
596 *
597 * MAX_SIZE should be as large as the number of css_ids. Ideally, we could get
598 * this constant directly from cgroup, but it is understandable that this is
599 * better kept as an internal representation in cgroup.c. In any case, the
600 * css_id space is not getting any smaller, and we don't have to necessarily
601 * increase ours as well if it increases.
602 */
603#define MEMCG_CACHES_MIN_SIZE 4
604#define MEMCG_CACHES_MAX_SIZE 65535
605
Glauber Costad7f25f82012-12-18 14:22:40 -0800606/*
607 * A lot of the calls to the cache allocation functions are expected to be
608 * inlined by the compiler. Since the calls to memcg_kmem_get_cache are
609 * conditional to this static branch, we'll have to allow modules that does
610 * kmem_cache_alloc and the such to see this symbol as well
611 */
Glauber Costaa8964b92012-12-18 14:22:09 -0800612struct static_key memcg_kmem_enabled_key;
Glauber Costad7f25f82012-12-18 14:22:40 -0800613EXPORT_SYMBOL(memcg_kmem_enabled_key);
Glauber Costaa8964b92012-12-18 14:22:09 -0800614
615static void disarm_kmem_keys(struct mem_cgroup *memcg)
616{
Glauber Costa55007d82012-12-18 14:22:38 -0800617 if (memcg_kmem_is_active(memcg)) {
Glauber Costaa8964b92012-12-18 14:22:09 -0800618 static_key_slow_dec(&memcg_kmem_enabled_key);
Glauber Costa55007d82012-12-18 14:22:38 -0800619 ida_simple_remove(&kmem_limited_groups, memcg->kmemcg_id);
620 }
Glauber Costabea207c2012-12-18 14:22:11 -0800621 /*
622 * This check can't live in kmem destruction function,
623 * since the charges will outlive the cgroup
624 */
625 WARN_ON(res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0);
Glauber Costaa8964b92012-12-18 14:22:09 -0800626}
627#else
628static void disarm_kmem_keys(struct mem_cgroup *memcg)
629{
630}
631#endif /* CONFIG_MEMCG_KMEM */
632
633static void disarm_static_keys(struct mem_cgroup *memcg)
634{
635 disarm_sock_keys(memcg);
636 disarm_kmem_keys(memcg);
637}
638
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700639static void drain_all_stock_async(struct mem_cgroup *memcg);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -0800640
Balbir Singhf64c3f52009-09-23 15:56:37 -0700641static struct mem_cgroup_per_zone *
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700642mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700643{
Glauber Costa45cf7eb2013-02-22 16:34:49 -0800644 VM_BUG_ON((unsigned)nid >= nr_node_ids);
Johannes Weiner54f72fe2013-07-08 15:59:49 -0700645 return &memcg->nodeinfo[nid]->zoneinfo[zid];
Balbir Singhf64c3f52009-09-23 15:56:37 -0700646}
647
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700648struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
Wu Fengguangd3242362009-12-16 12:19:59 +0100649{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700650 return &memcg->css;
Wu Fengguangd3242362009-12-16 12:19:59 +0100651}
652
Balbir Singhf64c3f52009-09-23 15:56:37 -0700653static struct mem_cgroup_per_zone *
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700654page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
Balbir Singhf64c3f52009-09-23 15:56:37 -0700655{
Johannes Weiner97a6c372011-03-23 16:42:27 -0700656 int nid = page_to_nid(page);
657 int zid = page_zonenum(page);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700658
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700659 return mem_cgroup_zoneinfo(memcg, nid, zid);
Balbir Singhf64c3f52009-09-23 15:56:37 -0700660}
661
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -0700662static struct mem_cgroup_tree_per_zone *
663soft_limit_tree_node_zone(int nid, int zid)
664{
665 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
666}
667
668static struct mem_cgroup_tree_per_zone *
669soft_limit_tree_from_page(struct page *page)
670{
671 int nid = page_to_nid(page);
672 int zid = page_zonenum(page);
673
674 return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
675}
676
677static void
678__mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
679 struct mem_cgroup_per_zone *mz,
680 struct mem_cgroup_tree_per_zone *mctz,
681 unsigned long long new_usage_in_excess)
682{
683 struct rb_node **p = &mctz->rb_root.rb_node;
684 struct rb_node *parent = NULL;
685 struct mem_cgroup_per_zone *mz_node;
686
687 if (mz->on_tree)
688 return;
689
690 mz->usage_in_excess = new_usage_in_excess;
691 if (!mz->usage_in_excess)
692 return;
693 while (*p) {
694 parent = *p;
695 mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
696 tree_node);
697 if (mz->usage_in_excess < mz_node->usage_in_excess)
698 p = &(*p)->rb_left;
699 /*
700 * We can't avoid mem cgroups that are over their soft
701 * limit by the same amount
702 */
703 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
704 p = &(*p)->rb_right;
705 }
706 rb_link_node(&mz->tree_node, parent, p);
707 rb_insert_color(&mz->tree_node, &mctz->rb_root);
708 mz->on_tree = true;
709}
710
711static void
712__mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
713 struct mem_cgroup_per_zone *mz,
714 struct mem_cgroup_tree_per_zone *mctz)
715{
716 if (!mz->on_tree)
717 return;
718 rb_erase(&mz->tree_node, &mctz->rb_root);
719 mz->on_tree = false;
720}
721
722static void
723mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
724 struct mem_cgroup_per_zone *mz,
725 struct mem_cgroup_tree_per_zone *mctz)
726{
727 spin_lock(&mctz->lock);
728 __mem_cgroup_remove_exceeded(memcg, mz, mctz);
729 spin_unlock(&mctz->lock);
730}
731
732
733static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
734{
735 unsigned long long excess;
736 struct mem_cgroup_per_zone *mz;
737 struct mem_cgroup_tree_per_zone *mctz;
738 int nid = page_to_nid(page);
739 int zid = page_zonenum(page);
740 mctz = soft_limit_tree_from_page(page);
741
742 /*
743 * Necessary to update all ancestors when hierarchy is used.
744 * because their event counter is not touched.
745 */
746 for (; memcg; memcg = parent_mem_cgroup(memcg)) {
747 mz = mem_cgroup_zoneinfo(memcg, nid, zid);
748 excess = res_counter_soft_limit_excess(&memcg->res);
749 /*
750 * We have to update the tree if mz is on RB-tree or
751 * mem is over its softlimit.
752 */
753 if (excess || mz->on_tree) {
754 spin_lock(&mctz->lock);
755 /* if on-tree, remove it */
756 if (mz->on_tree)
757 __mem_cgroup_remove_exceeded(memcg, mz, mctz);
758 /*
759 * Insert again. mz->usage_in_excess will be updated.
760 * If excess is 0, no tree ops.
761 */
762 __mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
763 spin_unlock(&mctz->lock);
764 }
765 }
766}
767
768static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
769{
770 int node, zone;
771 struct mem_cgroup_per_zone *mz;
772 struct mem_cgroup_tree_per_zone *mctz;
773
774 for_each_node(node) {
775 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
776 mz = mem_cgroup_zoneinfo(memcg, node, zone);
777 mctz = soft_limit_tree_node_zone(node, zone);
778 mem_cgroup_remove_exceeded(memcg, mz, mctz);
779 }
780 }
781}
782
783static struct mem_cgroup_per_zone *
784__mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
785{
786 struct rb_node *rightmost = NULL;
787 struct mem_cgroup_per_zone *mz;
788
789retry:
790 mz = NULL;
791 rightmost = rb_last(&mctz->rb_root);
792 if (!rightmost)
793 goto done; /* Nothing to reclaim from */
794
795 mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
796 /*
797 * Remove the node now but someone else can add it back,
798 * we will to add it back at the end of reclaim to its correct
799 * position in the tree.
800 */
801 __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
802 if (!res_counter_soft_limit_excess(&mz->memcg->res) ||
803 !css_tryget(&mz->memcg->css))
804 goto retry;
805done:
806 return mz;
807}
808
809static struct mem_cgroup_per_zone *
810mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
811{
812 struct mem_cgroup_per_zone *mz;
813
814 spin_lock(&mctz->lock);
815 mz = __mem_cgroup_largest_soft_limit_node(mctz);
816 spin_unlock(&mctz->lock);
817 return mz;
818}
819
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700820/*
821 * Implementation Note: reading percpu statistics for memcg.
822 *
823 * Both of vmstat[] and percpu_counter has threshold and do periodic
824 * synchronization to implement "quick" read. There are trade-off between
825 * reading cost and precision of value. Then, we may have a chance to implement
826 * a periodic synchronizion of counter in memcg's counter.
827 *
828 * But this _read() function is used for user interface now. The user accounts
829 * memory usage by memory cgroup and he _always_ requires exact value because
830 * he accounts memory. Even if we provide quick-and-fuzzy read, we always
831 * have to visit all online cpus and make sum. So, for now, unnecessary
832 * synchronization is not implemented. (just implemented for cpu hotplug)
833 *
834 * If there are kernel internal actions which can make use of some not-exact
835 * value, and reading all cpu value can be performance bottleneck in some
836 * common workload, threashold and synchonization as vmstat[] should be
837 * implemented.
838 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700839static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700840 enum mem_cgroup_stat_index idx)
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800841{
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700842 long val = 0;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800843 int cpu;
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800844
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700845 get_online_cpus();
846 for_each_online_cpu(cpu)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700847 val += per_cpu(memcg->stat->count[idx], cpu);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700848#ifdef CONFIG_HOTPLUG_CPU
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700849 spin_lock(&memcg->pcp_counter_lock);
850 val += memcg->nocpu_base.count[idx];
851 spin_unlock(&memcg->pcp_counter_lock);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -0700852#endif
853 put_online_cpus();
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800854 return val;
855}
856
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700857static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700858 bool charge)
859{
860 int val = (charge) ? 1 : -1;
Kamezawa Hiroyukibff6bb82012-07-31 16:41:38 -0700861 this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAP], val);
Balbir Singh0c3e73e2009-09-23 15:56:42 -0700862}
863
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700864static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
Johannes Weinere9f89742011-03-23 16:42:37 -0700865 enum mem_cgroup_events_index idx)
866{
867 unsigned long val = 0;
868 int cpu;
869
David Rientjes9c567512013-10-16 13:46:43 -0700870 get_online_cpus();
Johannes Weinere9f89742011-03-23 16:42:37 -0700871 for_each_online_cpu(cpu)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700872 val += per_cpu(memcg->stat->events[idx], cpu);
Johannes Weinere9f89742011-03-23 16:42:37 -0700873#ifdef CONFIG_HOTPLUG_CPU
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700874 spin_lock(&memcg->pcp_counter_lock);
875 val += memcg->nocpu_base.events[idx];
876 spin_unlock(&memcg->pcp_counter_lock);
Johannes Weinere9f89742011-03-23 16:42:37 -0700877#endif
David Rientjes9c567512013-10-16 13:46:43 -0700878 put_online_cpus();
Johannes Weinere9f89742011-03-23 16:42:37 -0700879 return val;
880}
881
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700882static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
David Rientjesb070e652013-05-07 16:18:09 -0700883 struct page *page,
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -0700884 bool anon, int nr_pages)
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800885{
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800886 preempt_disable();
887
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -0700888 /*
889 * Here, RSS means 'mapped anon' and anon's SwapCache. Shmem/tmpfs is
890 * counted as CACHE even if it's on ANON LRU.
891 */
892 if (anon)
893 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700894 nr_pages);
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800895 else
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -0700896 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700897 nr_pages);
Balaji Rao55e462b2008-05-01 04:35:12 -0700898
David Rientjesb070e652013-05-07 16:18:09 -0700899 if (PageTransHuge(page))
900 __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
901 nr_pages);
902
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800903 /* pagein of a big page is an event. So, ignore page size */
904 if (nr_pages > 0)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700905 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
KAMEZAWA Hiroyuki3751d602011-02-01 15:52:45 -0800906 else {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700907 __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
KAMEZAWA Hiroyuki3751d602011-02-01 15:52:45 -0800908 nr_pages = -nr_pages; /* for event */
909 }
KAMEZAWA Hiroyukie401f172011-01-20 14:44:23 -0800910
Johannes Weiner13114712012-05-29 15:07:07 -0700911 __this_cpu_add(memcg->stat->nr_page_events, nr_pages);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -0800912
KAMEZAWA Hiroyukic62b1a32010-03-10 15:22:29 -0800913 preempt_enable();
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800914}
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800915
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700916unsigned long
Hugh Dickins4d7dcca2012-05-29 15:07:08 -0700917mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
Konstantin Khlebnikov074291f2012-05-29 15:07:00 -0700918{
919 struct mem_cgroup_per_zone *mz;
920
921 mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
922 return mz->lru_size[lru];
923}
924
925static unsigned long
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700926mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700927 unsigned int lru_mask)
Ying Han889976d2011-05-26 16:25:33 -0700928{
929 struct mem_cgroup_per_zone *mz;
Hugh Dickinsf156ab92012-03-21 16:34:19 -0700930 enum lru_list lru;
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700931 unsigned long ret = 0;
932
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700933 mz = mem_cgroup_zoneinfo(memcg, nid, zid);
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700934
Hugh Dickinsf156ab92012-03-21 16:34:19 -0700935 for_each_lru(lru) {
936 if (BIT(lru) & lru_mask)
937 ret += mz->lru_size[lru];
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700938 }
939 return ret;
940}
941
942static unsigned long
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700943mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700944 int nid, unsigned int lru_mask)
945{
Ying Han889976d2011-05-26 16:25:33 -0700946 u64 total = 0;
947 int zid;
948
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700949 for (zid = 0; zid < MAX_NR_ZONES; zid++)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700950 total += mem_cgroup_zone_nr_lru_pages(memcg,
951 nid, zid, lru_mask);
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700952
Ying Han889976d2011-05-26 16:25:33 -0700953 return total;
954}
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700955
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700956static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -0700957 unsigned int lru_mask)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800958{
Ying Han889976d2011-05-26 16:25:33 -0700959 int nid;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800960 u64 total = 0;
961
Lai Jiangshan31aaea42012-12-12 13:51:27 -0800962 for_each_node_state(nid, N_MEMORY)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700963 total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -0800964 return total;
KAMEZAWA Hiroyukid52aa412008-02-07 00:14:24 -0800965}
966
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800967static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
968 enum mem_cgroup_events_target target)
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800969{
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700970 unsigned long val, next;
971
Johannes Weiner13114712012-05-29 15:07:07 -0700972 val = __this_cpu_read(memcg->stat->nr_page_events);
Steven Rostedt47994012011-11-02 13:38:33 -0700973 next = __this_cpu_read(memcg->stat->targets[target]);
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700974 /* from time_after() in jiffies.h */
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800975 if ((long)next - (long)val < 0) {
976 switch (target) {
977 case MEM_CGROUP_TARGET_THRESH:
978 next = val + THRESHOLDS_EVENTS_TARGET;
979 break;
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -0700980 case MEM_CGROUP_TARGET_SOFTLIMIT:
981 next = val + SOFTLIMIT_EVENTS_TARGET;
982 break;
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800983 case MEM_CGROUP_TARGET_NUMAINFO:
984 next = val + NUMAINFO_EVENTS_TARGET;
985 break;
986 default:
987 break;
988 }
989 __this_cpu_write(memcg->stat->targets[target], next);
990 return true;
Johannes Weiner7a159cc2011-03-23 16:42:38 -0700991 }
Johannes Weinerf53d7ce2012-01-12 17:18:23 -0800992 return false;
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -0800993}
994
995/*
996 * Check events in order.
997 *
998 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -0700999static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08001000{
Steven Rostedt47994012011-11-02 13:38:33 -07001001 preempt_disable();
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08001002 /* threshold event is triggered in finer grain than soft limit */
Johannes Weinerf53d7ce2012-01-12 17:18:23 -08001003 if (unlikely(mem_cgroup_event_ratelimit(memcg,
1004 MEM_CGROUP_TARGET_THRESH))) {
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -07001005 bool do_softlimit;
Andrew Morton82b3f2a2012-02-03 15:37:14 -08001006 bool do_numainfo __maybe_unused;
Johannes Weinerf53d7ce2012-01-12 17:18:23 -08001007
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -07001008 do_softlimit = mem_cgroup_event_ratelimit(memcg,
1009 MEM_CGROUP_TARGET_SOFTLIMIT);
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -07001010#if MAX_NUMNODES > 1
Johannes Weinerf53d7ce2012-01-12 17:18:23 -08001011 do_numainfo = mem_cgroup_event_ratelimit(memcg,
1012 MEM_CGROUP_TARGET_NUMAINFO);
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -07001013#endif
Johannes Weinerf53d7ce2012-01-12 17:18:23 -08001014 preempt_enable();
1015
1016 mem_cgroup_threshold(memcg);
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -07001017 if (unlikely(do_softlimit))
1018 mem_cgroup_update_tree(memcg, page);
Johannes Weinerf53d7ce2012-01-12 17:18:23 -08001019#if MAX_NUMNODES > 1
1020 if (unlikely(do_numainfo))
1021 atomic_inc(&memcg->numainfo_events);
1022#endif
1023 } else
1024 preempt_enable();
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08001025}
1026
Balbir Singhcf475ad2008-04-29 01:00:16 -07001027struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001028{
Balbir Singh31a78f22008-09-28 23:09:31 +01001029 /*
1030 * mm_update_next_owner() may clear mm->owner to NULL
1031 * if it races with swapoff, page migration, etc.
1032 * So this can be called with p == NULL.
1033 */
1034 if (unlikely(!p))
1035 return NULL;
1036
Tejun Heo8af01f52013-08-08 20:11:22 -04001037 return mem_cgroup_from_css(task_css(p, mem_cgroup_subsys_id));
Pavel Emelianov78fb7462008-02-07 00:13:51 -08001038}
1039
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07001040struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001041{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001042 struct mem_cgroup *memcg = NULL;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001043
1044 if (!mm)
1045 return NULL;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001046 /*
1047 * Because we have no locks, mm->owner's may be being moved to other
1048 * cgroup. We use css_tryget() here even if this looks
1049 * pessimistic (rather than adding locks here).
1050 */
1051 rcu_read_lock();
1052 do {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001053 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1054 if (unlikely(!memcg))
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001055 break;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001056 } while (!css_tryget(&memcg->css));
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001057 rcu_read_unlock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001058 return memcg;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08001059}
1060
Michal Hocko16248d82013-04-29 15:07:19 -07001061/*
1062 * Returns a next (in a pre-order walk) alive memcg (with elevated css
1063 * ref. count) or NULL if the whole root's subtree has been visited.
1064 *
1065 * helper function to be used by mem_cgroup_iter
1066 */
1067static struct mem_cgroup *__mem_cgroup_iter_next(struct mem_cgroup *root,
Andrew Morton694fbc02013-09-24 15:27:37 -07001068 struct mem_cgroup *last_visited)
Michal Hocko16248d82013-04-29 15:07:19 -07001069{
Tejun Heo492eb212013-08-08 20:11:25 -04001070 struct cgroup_subsys_state *prev_css, *next_css;
Michal Hocko16248d82013-04-29 15:07:19 -07001071
Tejun Heobd8815a2013-08-08 20:11:27 -04001072 prev_css = last_visited ? &last_visited->css : NULL;
Michal Hocko16248d82013-04-29 15:07:19 -07001073skip_node:
Tejun Heo492eb212013-08-08 20:11:25 -04001074 next_css = css_next_descendant_pre(prev_css, &root->css);
Michal Hocko16248d82013-04-29 15:07:19 -07001075
1076 /*
1077 * Even if we found a group we have to make sure it is
1078 * alive. css && !memcg means that the groups should be
1079 * skipped and we should continue the tree walk.
1080 * last_visited css is safe to use because it is
1081 * protected by css_get and the tree walk is rcu safe.
1082 */
Tejun Heo492eb212013-08-08 20:11:25 -04001083 if (next_css) {
1084 struct mem_cgroup *mem = mem_cgroup_from_css(next_css);
1085
Andrew Morton694fbc02013-09-24 15:27:37 -07001086 if (css_tryget(&mem->css))
1087 return mem;
1088 else {
Tejun Heo492eb212013-08-08 20:11:25 -04001089 prev_css = next_css;
Michal Hocko16248d82013-04-29 15:07:19 -07001090 goto skip_node;
1091 }
1092 }
1093
1094 return NULL;
1095}
1096
Johannes Weiner519ebea2013-07-03 15:04:51 -07001097static void mem_cgroup_iter_invalidate(struct mem_cgroup *root)
1098{
1099 /*
1100 * When a group in the hierarchy below root is destroyed, the
1101 * hierarchy iterator can no longer be trusted since it might
1102 * have pointed to the destroyed group. Invalidate it.
1103 */
1104 atomic_inc(&root->dead_count);
1105}
1106
1107static struct mem_cgroup *
1108mem_cgroup_iter_load(struct mem_cgroup_reclaim_iter *iter,
1109 struct mem_cgroup *root,
1110 int *sequence)
1111{
1112 struct mem_cgroup *position = NULL;
1113 /*
1114 * A cgroup destruction happens in two stages: offlining and
1115 * release. They are separated by a RCU grace period.
1116 *
1117 * If the iterator is valid, we may still race with an
1118 * offlining. The RCU lock ensures the object won't be
1119 * released, tryget will fail if we lost the race.
1120 */
1121 *sequence = atomic_read(&root->dead_count);
1122 if (iter->last_dead_count == *sequence) {
1123 smp_rmb();
1124 position = iter->last_visited;
1125 if (position && !css_tryget(&position->css))
1126 position = NULL;
1127 }
1128 return position;
1129}
1130
1131static void mem_cgroup_iter_update(struct mem_cgroup_reclaim_iter *iter,
1132 struct mem_cgroup *last_visited,
1133 struct mem_cgroup *new_position,
1134 int sequence)
1135{
1136 if (last_visited)
1137 css_put(&last_visited->css);
1138 /*
1139 * We store the sequence count from the time @last_visited was
1140 * loaded successfully instead of rereading it here so that we
1141 * don't lose destruction events in between. We could have
1142 * raced with the destruction of @new_position after all.
1143 */
1144 iter->last_visited = new_position;
1145 smp_wmb();
1146 iter->last_dead_count = sequence;
1147}
1148
Johannes Weiner56600482012-01-12 17:17:59 -08001149/**
1150 * mem_cgroup_iter - iterate over memory cgroup hierarchy
1151 * @root: hierarchy root
1152 * @prev: previously returned memcg, NULL on first invocation
1153 * @reclaim: cookie for shared reclaim walks, NULL for full walks
1154 *
1155 * Returns references to children of the hierarchy below @root, or
1156 * @root itself, or %NULL after a full round-trip.
1157 *
1158 * Caller must pass the return value in @prev on subsequent
1159 * invocations for reference counting, or use mem_cgroup_iter_break()
1160 * to cancel a hierarchy walk before the round-trip is complete.
1161 *
1162 * Reclaimers can specify a zone and a priority level in @reclaim to
1163 * divide up the memcgs in the hierarchy among all concurrent
1164 * reclaimers operating on the same zone and priority.
1165 */
Andrew Morton694fbc02013-09-24 15:27:37 -07001166struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
Johannes Weiner56600482012-01-12 17:17:59 -08001167 struct mem_cgroup *prev,
Andrew Morton694fbc02013-09-24 15:27:37 -07001168 struct mem_cgroup_reclaim_cookie *reclaim)
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07001169{
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001170 struct mem_cgroup *memcg = NULL;
Michal Hocko542f85f2013-04-29 15:07:15 -07001171 struct mem_cgroup *last_visited = NULL;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001172
Andrew Morton694fbc02013-09-24 15:27:37 -07001173 if (mem_cgroup_disabled())
1174 return NULL;
Johannes Weiner56600482012-01-12 17:17:59 -08001175
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07001176 if (!root)
1177 root = root_mem_cgroup;
1178
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001179 if (prev && !reclaim)
Michal Hocko542f85f2013-04-29 15:07:15 -07001180 last_visited = prev;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001181
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001182 if (!root->use_hierarchy && root != root_mem_cgroup) {
1183 if (prev)
Michal Hockoc40046f2013-04-29 15:07:14 -07001184 goto out_css_put;
Andrew Morton694fbc02013-09-24 15:27:37 -07001185 return root;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001186 }
1187
Michal Hocko542f85f2013-04-29 15:07:15 -07001188 rcu_read_lock();
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001189 while (!memcg) {
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001190 struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
Johannes Weiner519ebea2013-07-03 15:04:51 -07001191 int uninitialized_var(seq);
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001192
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001193 if (reclaim) {
1194 int nid = zone_to_nid(reclaim->zone);
1195 int zid = zone_idx(reclaim->zone);
1196 struct mem_cgroup_per_zone *mz;
1197
1198 mz = mem_cgroup_zoneinfo(root, nid, zid);
1199 iter = &mz->reclaim_iter[reclaim->priority];
Michal Hocko542f85f2013-04-29 15:07:15 -07001200 if (prev && reclaim->generation != iter->generation) {
Michal Hocko5f578162013-04-29 15:07:17 -07001201 iter->last_visited = NULL;
Michal Hocko542f85f2013-04-29 15:07:15 -07001202 goto out_unlock;
1203 }
Michal Hocko5f578162013-04-29 15:07:17 -07001204
Johannes Weiner519ebea2013-07-03 15:04:51 -07001205 last_visited = mem_cgroup_iter_load(iter, root, &seq);
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001206 }
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001207
Andrew Morton694fbc02013-09-24 15:27:37 -07001208 memcg = __mem_cgroup_iter_next(root, last_visited);
Michal Hocko542f85f2013-04-29 15:07:15 -07001209
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001210 if (reclaim) {
Johannes Weiner519ebea2013-07-03 15:04:51 -07001211 mem_cgroup_iter_update(iter, last_visited, memcg, seq);
Michal Hocko542f85f2013-04-29 15:07:15 -07001212
Michal Hocko19f39402013-04-29 15:07:18 -07001213 if (!memcg)
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001214 iter->generation++;
1215 else if (!prev && memcg)
1216 reclaim->generation = iter->generation;
1217 }
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001218
Andrew Morton694fbc02013-09-24 15:27:37 -07001219 if (prev && !memcg)
Michal Hocko542f85f2013-04-29 15:07:15 -07001220 goto out_unlock;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001221 }
Michal Hocko542f85f2013-04-29 15:07:15 -07001222out_unlock:
1223 rcu_read_unlock();
Michal Hockoc40046f2013-04-29 15:07:14 -07001224out_css_put:
1225 if (prev && prev != root)
1226 css_put(&prev->css);
1227
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001228 return memcg;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001229}
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001230
Johannes Weiner56600482012-01-12 17:17:59 -08001231/**
1232 * mem_cgroup_iter_break - abort a hierarchy walk prematurely
1233 * @root: hierarchy root
1234 * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
1235 */
1236void mem_cgroup_iter_break(struct mem_cgroup *root,
1237 struct mem_cgroup *prev)
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001238{
1239 if (!root)
1240 root = root_mem_cgroup;
1241 if (prev && prev != root)
1242 css_put(&prev->css);
1243}
1244
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001245/*
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001246 * Iteration constructs for visiting all cgroups (under a tree). If
1247 * loops are exited prematurely (break), mem_cgroup_iter_break() must
1248 * be used for reference counting.
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001249 */
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001250#define for_each_mem_cgroup_tree(iter, root) \
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001251 for (iter = mem_cgroup_iter(root, NULL, NULL); \
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001252 iter != NULL; \
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001253 iter = mem_cgroup_iter(root, iter, NULL))
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001254
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001255#define for_each_mem_cgroup(iter) \
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001256 for (iter = mem_cgroup_iter(NULL, NULL, NULL); \
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08001257 iter != NULL; \
Johannes Weiner527a5ec2012-01-12 17:17:55 -08001258 iter = mem_cgroup_iter(NULL, iter, NULL))
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001259
David Rientjes68ae5642012-12-12 13:51:57 -08001260void __mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
Ying Han456f9982011-05-26 16:25:38 -07001261{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001262 struct mem_cgroup *memcg;
Ying Han456f9982011-05-26 16:25:38 -07001263
Ying Han456f9982011-05-26 16:25:38 -07001264 rcu_read_lock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001265 memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
1266 if (unlikely(!memcg))
Ying Han456f9982011-05-26 16:25:38 -07001267 goto out;
1268
1269 switch (idx) {
Ying Han456f9982011-05-26 16:25:38 -07001270 case PGFAULT:
Johannes Weiner0e574a92012-01-12 17:18:35 -08001271 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
1272 break;
1273 case PGMAJFAULT:
1274 this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
Ying Han456f9982011-05-26 16:25:38 -07001275 break;
1276 default:
1277 BUG();
1278 }
1279out:
1280 rcu_read_unlock();
1281}
David Rientjes68ae5642012-12-12 13:51:57 -08001282EXPORT_SYMBOL(__mem_cgroup_count_vm_event);
Ying Han456f9982011-05-26 16:25:38 -07001283
Johannes Weiner925b7672012-01-12 17:18:15 -08001284/**
1285 * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
1286 * @zone: zone of the wanted lruvec
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001287 * @memcg: memcg of the wanted lruvec
Johannes Weiner925b7672012-01-12 17:18:15 -08001288 *
1289 * Returns the lru list vector holding pages for the given @zone and
1290 * @mem. This can be the global zone lruvec, if the memory controller
1291 * is disabled.
1292 */
1293struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
1294 struct mem_cgroup *memcg)
1295{
1296 struct mem_cgroup_per_zone *mz;
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001297 struct lruvec *lruvec;
Johannes Weiner925b7672012-01-12 17:18:15 -08001298
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001299 if (mem_cgroup_disabled()) {
1300 lruvec = &zone->lruvec;
1301 goto out;
1302 }
Johannes Weiner925b7672012-01-12 17:18:15 -08001303
1304 mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001305 lruvec = &mz->lruvec;
1306out:
1307 /*
1308 * Since a node can be onlined after the mem_cgroup was created,
1309 * we have to be prepared to initialize lruvec->zone here;
1310 * and if offlined then reonlined, we need to reinitialize it.
1311 */
1312 if (unlikely(lruvec->zone != zone))
1313 lruvec->zone = zone;
1314 return lruvec;
Johannes Weiner925b7672012-01-12 17:18:15 -08001315}
1316
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001317/*
1318 * Following LRU functions are allowed to be used without PCG_LOCK.
1319 * Operations are called by routine of global LRU independently from memcg.
1320 * What we have to take care of here is validness of pc->mem_cgroup.
1321 *
1322 * Changes to pc->mem_cgroup happens when
1323 * 1. charge
1324 * 2. moving account
1325 * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
1326 * It is added to LRU before charge.
1327 * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
1328 * When moving account, the page is not on LRU. It's isolated.
1329 */
1330
Johannes Weiner925b7672012-01-12 17:18:15 -08001331/**
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001332 * mem_cgroup_page_lruvec - return lruvec for adding an lru page
Johannes Weiner925b7672012-01-12 17:18:15 -08001333 * @page: the page
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001334 * @zone: zone of the page
Minchan Kim3f58a822011-03-22 16:32:53 -07001335 */
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001336struct lruvec *mem_cgroup_page_lruvec(struct page *page, struct zone *zone)
Minchan Kim3f58a822011-03-22 16:32:53 -07001337{
1338 struct mem_cgroup_per_zone *mz;
Johannes Weiner925b7672012-01-12 17:18:15 -08001339 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001340 struct page_cgroup *pc;
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001341 struct lruvec *lruvec;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08001342
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001343 if (mem_cgroup_disabled()) {
1344 lruvec = &zone->lruvec;
1345 goto out;
1346 }
Christoph Lameterb69408e2008-10-18 20:26:14 -07001347
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001348 pc = lookup_page_cgroup(page);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08001349 memcg = pc->mem_cgroup;
Hugh Dickins75121022012-03-05 14:59:18 -08001350
1351 /*
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001352 * Surreptitiously switch any uncharged offlist page to root:
Hugh Dickins75121022012-03-05 14:59:18 -08001353 * an uncharged page off lru does nothing to secure
1354 * its former mem_cgroup from sudden removal.
1355 *
1356 * Our caller holds lru_lock, and PageCgroupUsed is updated
1357 * under page_cgroup lock: between them, they make all uses
1358 * of pc->mem_cgroup safe.
1359 */
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001360 if (!PageLRU(page) && !PageCgroupUsed(pc) && memcg != root_mem_cgroup)
Hugh Dickins75121022012-03-05 14:59:18 -08001361 pc->mem_cgroup = memcg = root_mem_cgroup;
1362
Johannes Weiner925b7672012-01-12 17:18:15 -08001363 mz = page_cgroup_zoneinfo(memcg, page);
Hugh Dickinsbea8c152012-11-16 14:14:54 -08001364 lruvec = &mz->lruvec;
1365out:
1366 /*
1367 * Since a node can be onlined after the mem_cgroup was created,
1368 * we have to be prepared to initialize lruvec->zone here;
1369 * and if offlined then reonlined, we need to reinitialize it.
1370 */
1371 if (unlikely(lruvec->zone != zone))
1372 lruvec->zone = zone;
1373 return lruvec;
Johannes Weiner925b7672012-01-12 17:18:15 -08001374}
1375
1376/**
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001377 * mem_cgroup_update_lru_size - account for adding or removing an lru page
1378 * @lruvec: mem_cgroup per zone lru vector
1379 * @lru: index of lru list the page is sitting on
1380 * @nr_pages: positive when adding or negative when removing
Johannes Weiner925b7672012-01-12 17:18:15 -08001381 *
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001382 * This function must be called when a page is added to or removed from an
1383 * lru list.
Johannes Weiner925b7672012-01-12 17:18:15 -08001384 */
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001385void mem_cgroup_update_lru_size(struct lruvec *lruvec, enum lru_list lru,
1386 int nr_pages)
Johannes Weiner925b7672012-01-12 17:18:15 -08001387{
1388 struct mem_cgroup_per_zone *mz;
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001389 unsigned long *lru_size;
Johannes Weiner925b7672012-01-12 17:18:15 -08001390
1391 if (mem_cgroup_disabled())
1392 return;
1393
Hugh Dickinsfa9add62012-05-29 15:07:09 -07001394 mz = container_of(lruvec, struct mem_cgroup_per_zone, lruvec);
1395 lru_size = mz->lru_size + lru;
1396 *lru_size += nr_pages;
1397 VM_BUG_ON((long)(*lru_size) < 0);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001398}
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08001399
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08001400/*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001401 * Checks whether given mem is same or in the root_mem_cgroup's
Michal Hocko3e920412011-07-26 16:08:29 -07001402 * hierarchy subtree
1403 */
Johannes Weinerc3ac9a82012-05-29 15:06:25 -07001404bool __mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
1405 struct mem_cgroup *memcg)
Michal Hocko3e920412011-07-26 16:08:29 -07001406{
Johannes Weiner91c637342012-05-29 15:06:24 -07001407 if (root_memcg == memcg)
1408 return true;
Hugh Dickins3a981f42012-06-20 12:52:58 -07001409 if (!root_memcg->use_hierarchy || !memcg)
Johannes Weiner91c637342012-05-29 15:06:24 -07001410 return false;
Johannes Weinerc3ac9a82012-05-29 15:06:25 -07001411 return css_is_ancestor(&memcg->css, &root_memcg->css);
1412}
1413
1414static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
1415 struct mem_cgroup *memcg)
1416{
1417 bool ret;
1418
Johannes Weiner91c637342012-05-29 15:06:24 -07001419 rcu_read_lock();
Johannes Weinerc3ac9a82012-05-29 15:06:25 -07001420 ret = __mem_cgroup_same_or_subtree(root_memcg, memcg);
Johannes Weiner91c637342012-05-29 15:06:24 -07001421 rcu_read_unlock();
1422 return ret;
Michal Hocko3e920412011-07-26 16:08:29 -07001423}
1424
David Rientjesffbdccf2013-07-03 15:01:23 -07001425bool task_in_mem_cgroup(struct task_struct *task,
1426 const struct mem_cgroup *memcg)
David Rientjes4c4a2212008-02-07 00:14:06 -08001427{
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001428 struct mem_cgroup *curr = NULL;
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -07001429 struct task_struct *p;
David Rientjesffbdccf2013-07-03 15:01:23 -07001430 bool ret;
David Rientjes4c4a2212008-02-07 00:14:06 -08001431
KAMEZAWA Hiroyuki158e0a22010-08-10 18:03:00 -07001432 p = find_lock_task_mm(task);
David Rientjesde077d22012-01-12 17:18:52 -08001433 if (p) {
1434 curr = try_get_mem_cgroup_from_mm(p->mm);
1435 task_unlock(p);
1436 } else {
1437 /*
1438 * All threads may have already detached their mm's, but the oom
1439 * killer still needs to detect if they have already been oom
1440 * killed to prevent needlessly killing additional tasks.
1441 */
David Rientjesffbdccf2013-07-03 15:01:23 -07001442 rcu_read_lock();
David Rientjesde077d22012-01-12 17:18:52 -08001443 curr = mem_cgroup_from_task(task);
1444 if (curr)
1445 css_get(&curr->css);
David Rientjesffbdccf2013-07-03 15:01:23 -07001446 rcu_read_unlock();
David Rientjesde077d22012-01-12 17:18:52 -08001447 }
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001448 if (!curr)
David Rientjesffbdccf2013-07-03 15:01:23 -07001449 return false;
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -08001450 /*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001451 * We should check use_hierarchy of "memcg" not "curr". Because checking
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -08001452 * use_hierarchy of "curr" here make this function true if hierarchy is
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001453 * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
1454 * hierarchy(even if use_hierarchy is disabled in "memcg").
Daisuke Nishimurad31f56d2009-12-15 16:47:12 -08001455 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001456 ret = mem_cgroup_same_or_subtree(memcg, curr);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07001457 css_put(&curr->css);
David Rientjes4c4a2212008-02-07 00:14:06 -08001458 return ret;
1459}
1460
Konstantin Khlebnikovc56d5c72012-05-29 15:07:00 -07001461int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001462{
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08001463 unsigned long inactive_ratio;
Johannes Weiner9b272972011-11-02 13:38:23 -07001464 unsigned long inactive;
1465 unsigned long active;
1466 unsigned long gb;
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001467
Hugh Dickins4d7dcca2012-05-29 15:07:08 -07001468 inactive = mem_cgroup_get_lru_size(lruvec, LRU_INACTIVE_ANON);
1469 active = mem_cgroup_get_lru_size(lruvec, LRU_ACTIVE_ANON);
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08001470
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08001471 gb = (inactive + active) >> (30 - PAGE_SHIFT);
1472 if (gb)
1473 inactive_ratio = int_sqrt(10 * gb);
1474 else
1475 inactive_ratio = 1;
1476
Johannes Weiner9b272972011-11-02 13:38:23 -07001477 return inactive * inactive_ratio < active;
KOSAKI Motohiroc772be92009-01-07 18:08:25 -08001478}
1479
Balbir Singh6d61ef42009-01-07 18:08:06 -08001480#define mem_cgroup_from_res_counter(counter, member) \
1481 container_of(counter, struct mem_cgroup, member)
1482
Johannes Weiner19942822011-02-01 15:52:43 -08001483/**
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001484 * mem_cgroup_margin - calculate chargeable space of a memory cgroup
Wanpeng Lidad75572012-06-20 12:53:01 -07001485 * @memcg: the memory cgroup
Johannes Weiner19942822011-02-01 15:52:43 -08001486 *
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001487 * Returns the maximum amount of memory @mem can be charged with, in
Johannes Weiner7ec99d62011-03-23 16:42:36 -07001488 * pages.
Johannes Weiner19942822011-02-01 15:52:43 -08001489 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001490static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
Johannes Weiner19942822011-02-01 15:52:43 -08001491{
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001492 unsigned long long margin;
1493
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001494 margin = res_counter_margin(&memcg->res);
Johannes Weiner9d11ea92011-03-23 16:42:21 -07001495 if (do_swap_account)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001496 margin = min(margin, res_counter_margin(&memcg->memsw));
Johannes Weiner7ec99d62011-03-23 16:42:36 -07001497 return margin >> PAGE_SHIFT;
Johannes Weiner19942822011-02-01 15:52:43 -08001498}
1499
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07001500int mem_cgroup_swappiness(struct mem_cgroup *memcg)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001501{
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001502 /* root ? */
Tejun Heo63876982013-08-08 20:11:23 -04001503 if (!css_parent(&memcg->css))
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001504 return vm_swappiness;
1505
Johannes Weinerbf1ff262011-03-23 16:42:32 -07001506 return memcg->swappiness;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08001507}
1508
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001509/*
1510 * memcg->moving_account is used for checking possibility that some thread is
1511 * calling move_account(). When a thread on CPU-A starts moving pages under
1512 * a memcg, other threads should check memcg->moving_account under
1513 * rcu_read_lock(), like this:
1514 *
1515 * CPU-A CPU-B
1516 * rcu_read_lock()
1517 * memcg->moving_account+1 if (memcg->mocing_account)
1518 * take heavy locks.
1519 * synchronize_rcu() update something.
1520 * rcu_read_unlock()
1521 * start move here.
1522 */
KAMEZAWA Hiroyuki4331f7d2012-03-21 16:34:26 -07001523
1524/* for quick checking without looking up memcg */
1525atomic_t memcg_moving __read_mostly;
1526
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001527static void mem_cgroup_start_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001528{
KAMEZAWA Hiroyuki4331f7d2012-03-21 16:34:26 -07001529 atomic_inc(&memcg_moving);
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001530 atomic_inc(&memcg->moving_account);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001531 synchronize_rcu();
1532}
1533
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001534static void mem_cgroup_end_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001535{
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001536 /*
1537 * Now, mem_cgroup_clear_mc() may call this function with NULL.
1538 * We check NULL in callee rather than caller.
1539 */
KAMEZAWA Hiroyuki4331f7d2012-03-21 16:34:26 -07001540 if (memcg) {
1541 atomic_dec(&memcg_moving);
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001542 atomic_dec(&memcg->moving_account);
KAMEZAWA Hiroyuki4331f7d2012-03-21 16:34:26 -07001543 }
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001544}
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001545
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001546/*
1547 * 2 routines for checking "mem" is under move_account() or not.
1548 *
Andrew Morton13fd1dd92012-03-21 16:34:26 -07001549 * mem_cgroup_stolen() - checking whether a cgroup is mc.from or not. This
1550 * is used for avoiding races in accounting. If true,
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001551 * pc->mem_cgroup may be overwritten.
1552 *
1553 * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
1554 * under hierarchy of moving cgroups. This is for
1555 * waiting at hith-memory prressure caused by "move".
1556 */
1557
Andrew Morton13fd1dd92012-03-21 16:34:26 -07001558static bool mem_cgroup_stolen(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001559{
1560 VM_BUG_ON(!rcu_read_lock_held());
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07001561 return atomic_read(&memcg->moving_account) > 0;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07001562}
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001563
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001564static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001565{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001566 struct mem_cgroup *from;
1567 struct mem_cgroup *to;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001568 bool ret = false;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001569 /*
1570 * Unlike task_move routines, we access mc.to, mc.from not under
1571 * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1572 */
1573 spin_lock(&mc.lock);
1574 from = mc.from;
1575 to = mc.to;
1576 if (!from)
1577 goto unlock;
Michal Hocko3e920412011-07-26 16:08:29 -07001578
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001579 ret = mem_cgroup_same_or_subtree(memcg, from)
1580 || mem_cgroup_same_or_subtree(memcg, to);
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07001581unlock:
1582 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001583 return ret;
1584}
1585
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001586static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001587{
1588 if (mc.moving_task && current != mc.moving_task) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001589 if (mem_cgroup_under_move(memcg)) {
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07001590 DEFINE_WAIT(wait);
1591 prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1592 /* moving charge context might have finished. */
1593 if (mc.moving_task)
1594 schedule();
1595 finish_wait(&mc.waitq, &wait);
1596 return true;
1597 }
1598 }
1599 return false;
1600}
1601
KAMEZAWA Hiroyuki312734c02012-03-21 16:34:24 -07001602/*
1603 * Take this lock when
1604 * - a code tries to modify page's memcg while it's USED.
1605 * - a code tries to modify page state accounting in a memcg.
Andrew Morton13fd1dd92012-03-21 16:34:26 -07001606 * see mem_cgroup_stolen(), too.
KAMEZAWA Hiroyuki312734c02012-03-21 16:34:24 -07001607 */
1608static void move_lock_mem_cgroup(struct mem_cgroup *memcg,
1609 unsigned long *flags)
1610{
1611 spin_lock_irqsave(&memcg->move_lock, *flags);
1612}
1613
1614static void move_unlock_mem_cgroup(struct mem_cgroup *memcg,
1615 unsigned long *flags)
1616{
1617 spin_unlock_irqrestore(&memcg->move_lock, *flags);
1618}
1619
Sha Zhengju58cf1882013-02-22 16:32:05 -08001620#define K(x) ((x) << (PAGE_SHIFT-10))
Balbir Singhe2224322009-04-02 16:57:39 -07001621/**
Sha Zhengju58cf1882013-02-22 16:32:05 -08001622 * mem_cgroup_print_oom_info: Print OOM information relevant to memory controller.
Balbir Singhe2224322009-04-02 16:57:39 -07001623 * @memcg: The memory cgroup that went over limit
1624 * @p: Task that is going to be killed
1625 *
1626 * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1627 * enabled
1628 */
1629void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1630{
1631 struct cgroup *task_cgrp;
1632 struct cgroup *mem_cgrp;
1633 /*
1634 * Need a buffer in BSS, can't rely on allocations. The code relies
1635 * on the assumption that OOM is serialized for memory controller.
1636 * If this assumption is broken, revisit this code.
1637 */
1638 static char memcg_name[PATH_MAX];
1639 int ret;
Sha Zhengju58cf1882013-02-22 16:32:05 -08001640 struct mem_cgroup *iter;
1641 unsigned int i;
Balbir Singhe2224322009-04-02 16:57:39 -07001642
Sha Zhengju58cf1882013-02-22 16:32:05 -08001643 if (!p)
Balbir Singhe2224322009-04-02 16:57:39 -07001644 return;
1645
Balbir Singhe2224322009-04-02 16:57:39 -07001646 rcu_read_lock();
1647
1648 mem_cgrp = memcg->css.cgroup;
1649 task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1650
1651 ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1652 if (ret < 0) {
1653 /*
1654 * Unfortunately, we are unable to convert to a useful name
1655 * But we'll still print out the usage information
1656 */
1657 rcu_read_unlock();
1658 goto done;
1659 }
1660 rcu_read_unlock();
1661
Andrew Mortond0451972013-02-22 16:32:06 -08001662 pr_info("Task in %s killed", memcg_name);
Balbir Singhe2224322009-04-02 16:57:39 -07001663
1664 rcu_read_lock();
1665 ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1666 if (ret < 0) {
1667 rcu_read_unlock();
1668 goto done;
1669 }
1670 rcu_read_unlock();
1671
1672 /*
1673 * Continues from above, so we don't need an KERN_ level
1674 */
Andrew Mortond0451972013-02-22 16:32:06 -08001675 pr_cont(" as a result of limit of %s\n", memcg_name);
Balbir Singhe2224322009-04-02 16:57:39 -07001676done:
1677
Andrew Mortond0451972013-02-22 16:32:06 -08001678 pr_info("memory: usage %llukB, limit %llukB, failcnt %llu\n",
Balbir Singhe2224322009-04-02 16:57:39 -07001679 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1680 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1681 res_counter_read_u64(&memcg->res, RES_FAILCNT));
Andrew Mortond0451972013-02-22 16:32:06 -08001682 pr_info("memory+swap: usage %llukB, limit %llukB, failcnt %llu\n",
Balbir Singhe2224322009-04-02 16:57:39 -07001683 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1684 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1685 res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
Andrew Mortond0451972013-02-22 16:32:06 -08001686 pr_info("kmem: usage %llukB, limit %llukB, failcnt %llu\n",
Glauber Costa510fc4e2012-12-18 14:21:47 -08001687 res_counter_read_u64(&memcg->kmem, RES_USAGE) >> 10,
1688 res_counter_read_u64(&memcg->kmem, RES_LIMIT) >> 10,
1689 res_counter_read_u64(&memcg->kmem, RES_FAILCNT));
Sha Zhengju58cf1882013-02-22 16:32:05 -08001690
1691 for_each_mem_cgroup_tree(iter, memcg) {
1692 pr_info("Memory cgroup stats");
1693
1694 rcu_read_lock();
1695 ret = cgroup_path(iter->css.cgroup, memcg_name, PATH_MAX);
1696 if (!ret)
1697 pr_cont(" for %s", memcg_name);
1698 rcu_read_unlock();
1699 pr_cont(":");
1700
1701 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
1702 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
1703 continue;
1704 pr_cont(" %s:%ldKB", mem_cgroup_stat_names[i],
1705 K(mem_cgroup_read_stat(iter, i)));
1706 }
1707
1708 for (i = 0; i < NR_LRU_LISTS; i++)
1709 pr_cont(" %s:%luKB", mem_cgroup_lru_names[i],
1710 K(mem_cgroup_nr_lru_pages(iter, BIT(i))));
1711
1712 pr_cont("\n");
1713 }
Balbir Singhe2224322009-04-02 16:57:39 -07001714}
1715
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001716/*
1717 * This function returns the number of memcg under hierarchy tree. Returns
1718 * 1(self count) if no children.
1719 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001720static int mem_cgroup_count_children(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001721{
1722 int num = 0;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001723 struct mem_cgroup *iter;
1724
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001725 for_each_mem_cgroup_tree(iter, memcg)
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07001726 num++;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07001727 return num;
1728}
1729
Balbir Singh6d61ef42009-01-07 18:08:06 -08001730/*
David Rientjesa63d83f2010-08-09 17:19:46 -07001731 * Return the memory (and swap, if configured) limit for a memcg.
1732 */
David Rientjes9cbb78b2012-07-31 16:43:44 -07001733static u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
David Rientjesa63d83f2010-08-09 17:19:46 -07001734{
1735 u64 limit;
David Rientjesa63d83f2010-08-09 17:19:46 -07001736
Johannes Weinerf3e8eb72011-01-13 15:47:39 -08001737 limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
Johannes Weinerf3e8eb72011-01-13 15:47:39 -08001738
David Rientjesa63d83f2010-08-09 17:19:46 -07001739 /*
Michal Hocko9a5a8f12012-11-16 14:14:49 -08001740 * Do not consider swap space if we cannot swap due to swappiness
David Rientjesa63d83f2010-08-09 17:19:46 -07001741 */
Michal Hocko9a5a8f12012-11-16 14:14:49 -08001742 if (mem_cgroup_swappiness(memcg)) {
1743 u64 memsw;
1744
1745 limit += total_swap_pages << PAGE_SHIFT;
1746 memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1747
1748 /*
1749 * If memsw is finite and limits the amount of swap space
1750 * available to this memcg, return that limit.
1751 */
1752 limit = min(limit, memsw);
1753 }
1754
1755 return limit;
David Rientjesa63d83f2010-08-09 17:19:46 -07001756}
1757
David Rientjes19965462012-12-11 16:00:26 -08001758static void mem_cgroup_out_of_memory(struct mem_cgroup *memcg, gfp_t gfp_mask,
1759 int order)
David Rientjes9cbb78b2012-07-31 16:43:44 -07001760{
1761 struct mem_cgroup *iter;
1762 unsigned long chosen_points = 0;
1763 unsigned long totalpages;
1764 unsigned int points = 0;
1765 struct task_struct *chosen = NULL;
1766
David Rientjes876aafb2012-07-31 16:43:48 -07001767 /*
David Rientjes465adcf2013-04-29 15:08:45 -07001768 * If current has a pending SIGKILL or is exiting, then automatically
1769 * select it. The goal is to allow it to allocate so that it may
1770 * quickly exit and free its memory.
David Rientjes876aafb2012-07-31 16:43:48 -07001771 */
David Rientjes465adcf2013-04-29 15:08:45 -07001772 if (fatal_signal_pending(current) || current->flags & PF_EXITING) {
David Rientjes876aafb2012-07-31 16:43:48 -07001773 set_thread_flag(TIF_MEMDIE);
1774 return;
1775 }
1776
1777 check_panic_on_oom(CONSTRAINT_MEMCG, gfp_mask, order, NULL);
David Rientjes9cbb78b2012-07-31 16:43:44 -07001778 totalpages = mem_cgroup_get_limit(memcg) >> PAGE_SHIFT ? : 1;
1779 for_each_mem_cgroup_tree(iter, memcg) {
Tejun Heo72ec7022013-08-08 20:11:26 -04001780 struct css_task_iter it;
David Rientjes9cbb78b2012-07-31 16:43:44 -07001781 struct task_struct *task;
1782
Tejun Heo72ec7022013-08-08 20:11:26 -04001783 css_task_iter_start(&iter->css, &it);
1784 while ((task = css_task_iter_next(&it))) {
David Rientjes9cbb78b2012-07-31 16:43:44 -07001785 switch (oom_scan_process_thread(task, totalpages, NULL,
1786 false)) {
1787 case OOM_SCAN_SELECT:
1788 if (chosen)
1789 put_task_struct(chosen);
1790 chosen = task;
1791 chosen_points = ULONG_MAX;
1792 get_task_struct(chosen);
1793 /* fall through */
1794 case OOM_SCAN_CONTINUE:
1795 continue;
1796 case OOM_SCAN_ABORT:
Tejun Heo72ec7022013-08-08 20:11:26 -04001797 css_task_iter_end(&it);
David Rientjes9cbb78b2012-07-31 16:43:44 -07001798 mem_cgroup_iter_break(memcg, iter);
1799 if (chosen)
1800 put_task_struct(chosen);
1801 return;
1802 case OOM_SCAN_OK:
1803 break;
1804 };
1805 points = oom_badness(task, memcg, NULL, totalpages);
1806 if (points > chosen_points) {
1807 if (chosen)
1808 put_task_struct(chosen);
1809 chosen = task;
1810 chosen_points = points;
1811 get_task_struct(chosen);
1812 }
1813 }
Tejun Heo72ec7022013-08-08 20:11:26 -04001814 css_task_iter_end(&it);
David Rientjes9cbb78b2012-07-31 16:43:44 -07001815 }
1816
1817 if (!chosen)
1818 return;
1819 points = chosen_points * 1000 / totalpages;
David Rientjes9cbb78b2012-07-31 16:43:44 -07001820 oom_kill_process(chosen, gfp_mask, order, points, totalpages, memcg,
1821 NULL, "Memory cgroup out of memory");
David Rientjes9cbb78b2012-07-31 16:43:44 -07001822}
1823
Johannes Weiner56600482012-01-12 17:17:59 -08001824static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
1825 gfp_t gfp_mask,
1826 unsigned long flags)
1827{
1828 unsigned long total = 0;
1829 bool noswap = false;
1830 int loop;
1831
1832 if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
1833 noswap = true;
1834 if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
1835 noswap = true;
1836
1837 for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
1838 if (loop)
1839 drain_all_stock_async(memcg);
1840 total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
1841 /*
1842 * Allow limit shrinkers, which are triggered directly
1843 * by userspace, to catch signals and stop reclaim
1844 * after minimal progress, regardless of the margin.
1845 */
1846 if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
1847 break;
1848 if (mem_cgroup_margin(memcg))
1849 break;
1850 /*
1851 * If nothing was reclaimed after two attempts, there
1852 * may be no reclaimable pages in this hierarchy.
1853 */
1854 if (loop && !total)
1855 break;
1856 }
1857 return total;
1858}
1859
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001860/**
1861 * test_mem_cgroup_node_reclaimable
Wanpeng Lidad75572012-06-20 12:53:01 -07001862 * @memcg: the target memcg
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001863 * @nid: the node ID to be checked.
1864 * @noswap : specify true here if the user wants flle only information.
1865 *
1866 * This function returns whether the specified memcg contains any
1867 * reclaimable pages on a node. Returns true if there are any reclaimable
1868 * pages in the node.
1869 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001870static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001871 int nid, bool noswap)
1872{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001873 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001874 return true;
1875 if (noswap || !total_swap_pages)
1876 return false;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001877 if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001878 return true;
1879 return false;
1880
1881}
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -07001882#if MAX_NUMNODES > 1
Ying Han889976d2011-05-26 16:25:33 -07001883
1884/*
1885 * Always updating the nodemask is not very good - even if we have an empty
1886 * list or the wrong list here, we can start from some node and traverse all
1887 * nodes based on the zonelist. So update the list loosely once per 10 secs.
1888 *
1889 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001890static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
Ying Han889976d2011-05-26 16:25:33 -07001891{
1892 int nid;
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -07001893 /*
1894 * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
1895 * pagein/pageout changes since the last update.
1896 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001897 if (!atomic_read(&memcg->numainfo_events))
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -07001898 return;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001899 if (atomic_inc_return(&memcg->numainfo_updating) > 1)
Ying Han889976d2011-05-26 16:25:33 -07001900 return;
1901
Ying Han889976d2011-05-26 16:25:33 -07001902 /* make a nodemask where this memcg uses memory from */
Lai Jiangshan31aaea42012-12-12 13:51:27 -08001903 memcg->scan_nodes = node_states[N_MEMORY];
Ying Han889976d2011-05-26 16:25:33 -07001904
Lai Jiangshan31aaea42012-12-12 13:51:27 -08001905 for_each_node_mask(nid, node_states[N_MEMORY]) {
Ying Han889976d2011-05-26 16:25:33 -07001906
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001907 if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
1908 node_clear(nid, memcg->scan_nodes);
Ying Han889976d2011-05-26 16:25:33 -07001909 }
KAMEZAWA Hiroyuki453a9bf2011-07-08 15:39:43 -07001910
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001911 atomic_set(&memcg->numainfo_events, 0);
1912 atomic_set(&memcg->numainfo_updating, 0);
Ying Han889976d2011-05-26 16:25:33 -07001913}
1914
1915/*
1916 * Selecting a node where we start reclaim from. Because what we need is just
1917 * reducing usage counter, start from anywhere is O,K. Considering
1918 * memory reclaim from current node, there are pros. and cons.
1919 *
1920 * Freeing memory from current node means freeing memory from a node which
1921 * we'll use or we've used. So, it may make LRU bad. And if several threads
1922 * hit limits, it will see a contention on a node. But freeing from remote
1923 * node means more costs for memory reclaim because of memory latency.
1924 *
1925 * Now, we use round-robin. Better algorithm is welcomed.
1926 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001927int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
Ying Han889976d2011-05-26 16:25:33 -07001928{
1929 int node;
1930
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001931 mem_cgroup_may_update_nodemask(memcg);
1932 node = memcg->last_scanned_node;
Ying Han889976d2011-05-26 16:25:33 -07001933
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001934 node = next_node(node, memcg->scan_nodes);
Ying Han889976d2011-05-26 16:25:33 -07001935 if (node == MAX_NUMNODES)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001936 node = first_node(memcg->scan_nodes);
Ying Han889976d2011-05-26 16:25:33 -07001937 /*
1938 * We call this when we hit limit, not when pages are added to LRU.
1939 * No LRU may hold pages because all pages are UNEVICTABLE or
1940 * memcg is too small and all pages are not on LRU. In that case,
1941 * we use curret node.
1942 */
1943 if (unlikely(node == MAX_NUMNODES))
1944 node = numa_node_id();
1945
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001946 memcg->last_scanned_node = node;
Ying Han889976d2011-05-26 16:25:33 -07001947 return node;
1948}
1949
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -07001950/*
1951 * Check all nodes whether it contains reclaimable pages or not.
1952 * For quick scan, we make use of scan_nodes. This will allow us to skip
1953 * unused nodes. But scan_nodes is lazily updated and may not cotain
1954 * enough new information. We need to do double check.
1955 */
1956static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1957{
1958 int nid;
1959
1960 /*
1961 * quick check...making use of scan_node.
1962 * We can skip unused nodes.
1963 */
1964 if (!nodes_empty(memcg->scan_nodes)) {
1965 for (nid = first_node(memcg->scan_nodes);
1966 nid < MAX_NUMNODES;
1967 nid = next_node(nid, memcg->scan_nodes)) {
1968
1969 if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1970 return true;
1971 }
1972 }
1973 /*
1974 * Check rest of nodes.
1975 */
1976 for_each_node_state(nid, N_MEMORY) {
1977 if (node_isset(nid, memcg->scan_nodes))
1978 continue;
1979 if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
1980 return true;
1981 }
1982 return false;
1983}
1984
Ying Han889976d2011-05-26 16:25:33 -07001985#else
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07001986int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
Ying Han889976d2011-05-26 16:25:33 -07001987{
1988 return 0;
1989}
KAMEZAWA Hiroyuki4d0c0662011-07-08 15:39:42 -07001990
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -07001991static bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
1992{
1993 return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
1994}
Ying Han889976d2011-05-26 16:25:33 -07001995#endif
1996
Andrew Morton0608f432013-09-24 15:27:41 -07001997static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
1998 struct zone *zone,
1999 gfp_t gfp_mask,
2000 unsigned long *total_scanned)
Balbir Singh6d61ef42009-01-07 18:08:06 -08002001{
Andrew Morton0608f432013-09-24 15:27:41 -07002002 struct mem_cgroup *victim = NULL;
2003 int total = 0;
2004 int loop = 0;
2005 unsigned long excess;
2006 unsigned long nr_scanned;
2007 struct mem_cgroup_reclaim_cookie reclaim = {
2008 .zone = zone,
2009 .priority = 0,
2010 };
Johannes Weiner9d11ea92011-03-23 16:42:21 -07002011
Andrew Morton0608f432013-09-24 15:27:41 -07002012 excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
Balbir Singh6d61ef42009-01-07 18:08:06 -08002013
Andrew Morton0608f432013-09-24 15:27:41 -07002014 while (1) {
2015 victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
2016 if (!victim) {
2017 loop++;
2018 if (loop >= 2) {
2019 /*
2020 * If we have not been able to reclaim
2021 * anything, it might because there are
2022 * no reclaimable pages under this hierarchy
2023 */
2024 if (!total)
2025 break;
2026 /*
2027 * We want to do more targeted reclaim.
2028 * excess >> 2 is not to excessive so as to
2029 * reclaim too much, nor too less that we keep
2030 * coming back to reclaim from this cgroup
2031 */
2032 if (total >= (excess >> 2) ||
2033 (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
2034 break;
2035 }
2036 continue;
2037 }
2038 if (!mem_cgroup_reclaimable(victim, false))
2039 continue;
2040 total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
2041 zone, &nr_scanned);
2042 *total_scanned += nr_scanned;
2043 if (!res_counter_soft_limit_excess(&root_memcg->res))
2044 break;
Balbir Singh6d61ef42009-01-07 18:08:06 -08002045 }
Andrew Morton0608f432013-09-24 15:27:41 -07002046 mem_cgroup_iter_break(root_memcg, victim);
2047 return total;
Balbir Singh6d61ef42009-01-07 18:08:06 -08002048}
2049
Johannes Weiner0056f4e2013-10-31 16:34:14 -07002050#ifdef CONFIG_LOCKDEP
2051static struct lockdep_map memcg_oom_lock_dep_map = {
2052 .name = "memcg_oom_lock",
2053};
2054#endif
2055
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002056static DEFINE_SPINLOCK(memcg_oom_lock);
2057
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002058/*
2059 * Check OOM-Killer is already running under our hierarchy.
2060 * If someone is running, return false.
2061 */
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002062static bool mem_cgroup_oom_trylock(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002063{
Michal Hocko79dfdac2011-07-26 16:08:23 -07002064 struct mem_cgroup *iter, *failed = NULL;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002065
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002066 spin_lock(&memcg_oom_lock);
2067
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08002068 for_each_mem_cgroup_tree(iter, memcg) {
Johannes Weiner23751be2011-08-25 15:59:16 -07002069 if (iter->oom_lock) {
Michal Hocko79dfdac2011-07-26 16:08:23 -07002070 /*
2071 * this subtree of our hierarchy is already locked
2072 * so we cannot give a lock.
2073 */
Michal Hocko79dfdac2011-07-26 16:08:23 -07002074 failed = iter;
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08002075 mem_cgroup_iter_break(memcg, iter);
2076 break;
Johannes Weiner23751be2011-08-25 15:59:16 -07002077 } else
2078 iter->oom_lock = true;
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07002079 }
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002080
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002081 if (failed) {
2082 /*
2083 * OK, we failed to lock the whole subtree so we have
2084 * to clean up what we set up to the failing subtree
2085 */
2086 for_each_mem_cgroup_tree(iter, memcg) {
2087 if (iter == failed) {
2088 mem_cgroup_iter_break(memcg, iter);
2089 break;
2090 }
2091 iter->oom_lock = false;
Michal Hocko79dfdac2011-07-26 16:08:23 -07002092 }
Johannes Weiner0056f4e2013-10-31 16:34:14 -07002093 } else
2094 mutex_acquire(&memcg_oom_lock_dep_map, 0, 1, _RET_IP_);
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002095
2096 spin_unlock(&memcg_oom_lock);
2097
2098 return !failed;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002099}
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07002100
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002101static void mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07002102{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07002103 struct mem_cgroup *iter;
2104
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002105 spin_lock(&memcg_oom_lock);
Johannes Weiner0056f4e2013-10-31 16:34:14 -07002106 mutex_release(&memcg_oom_lock_dep_map, 1, _RET_IP_);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002107 for_each_mem_cgroup_tree(iter, memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07002108 iter->oom_lock = false;
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002109 spin_unlock(&memcg_oom_lock);
Michal Hocko79dfdac2011-07-26 16:08:23 -07002110}
2111
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002112static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07002113{
2114 struct mem_cgroup *iter;
2115
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002116 for_each_mem_cgroup_tree(iter, memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07002117 atomic_inc(&iter->under_oom);
2118}
2119
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002120static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07002121{
2122 struct mem_cgroup *iter;
2123
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002124 /*
2125 * When a new child is created while the hierarchy is under oom,
2126 * mem_cgroup_oom_lock() may not be called. We have to use
2127 * atomic_add_unless() here.
2128 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002129 for_each_mem_cgroup_tree(iter, memcg)
Michal Hocko79dfdac2011-07-26 16:08:23 -07002130 atomic_add_unless(&iter->under_oom, -1, 0);
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07002131}
2132
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002133static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
2134
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002135struct oom_wait_info {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07002136 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002137 wait_queue_t wait;
2138};
2139
2140static int memcg_oom_wake_function(wait_queue_t *wait,
2141 unsigned mode, int sync, void *arg)
2142{
Hugh Dickinsd79154b2012-03-21 16:34:18 -07002143 struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg;
2144 struct mem_cgroup *oom_wait_memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002145 struct oom_wait_info *oom_wait_info;
2146
2147 oom_wait_info = container_of(wait, struct oom_wait_info, wait);
Hugh Dickinsd79154b2012-03-21 16:34:18 -07002148 oom_wait_memcg = oom_wait_info->memcg;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002149
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002150 /*
Hugh Dickinsd79154b2012-03-21 16:34:18 -07002151 * Both of oom_wait_info->memcg and wake_memcg are stable under us.
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002152 * Then we can use css_is_ancestor without taking care of RCU.
2153 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002154 if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
2155 && !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002156 return 0;
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002157 return autoremove_wake_function(wait, mode, sync, arg);
2158}
2159
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002160static void memcg_wakeup_oom(struct mem_cgroup *memcg)
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002161{
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002162 atomic_inc(&memcg->oom_wakeups);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002163 /* for filtering, pass "memcg" as argument. */
2164 __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
KAMEZAWA Hiroyukidc98df52010-05-26 14:42:36 -07002165}
2166
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002167static void memcg_oom_recover(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002168{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002169 if (memcg && atomic_read(&memcg->under_oom))
2170 memcg_wakeup_oom(memcg);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002171}
2172
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002173static void mem_cgroup_oom(struct mem_cgroup *memcg, gfp_t mask, int order)
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002174{
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002175 if (!current->memcg_oom.may_oom)
2176 return;
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002177 /*
Johannes Weiner49426422013-10-16 13:46:59 -07002178 * We are in the middle of the charge context here, so we
2179 * don't want to block when potentially sitting on a callstack
2180 * that holds all kinds of filesystem and mm locks.
2181 *
2182 * Also, the caller may handle a failed allocation gracefully
2183 * (like optional page cache readahead) and so an OOM killer
2184 * invocation might not even be necessary.
2185 *
2186 * That's why we don't do anything here except remember the
2187 * OOM context and then deal with it at the end of the page
2188 * fault when the stack is unwound, the locks are released,
2189 * and when we know whether the fault was overall successful.
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002190 */
Johannes Weiner49426422013-10-16 13:46:59 -07002191 css_get(&memcg->css);
2192 current->memcg_oom.memcg = memcg;
2193 current->memcg_oom.gfp_mask = mask;
2194 current->memcg_oom.order = order;
2195}
2196
2197/**
2198 * mem_cgroup_oom_synchronize - complete memcg OOM handling
2199 * @handle: actually kill/wait or just clean up the OOM state
2200 *
2201 * This has to be called at the end of a page fault if the memcg OOM
2202 * handler was enabled.
2203 *
2204 * Memcg supports userspace OOM handling where failed allocations must
2205 * sleep on a waitqueue until the userspace task resolves the
2206 * situation. Sleeping directly in the charge context with all kinds
2207 * of locks held is not a good idea, instead we remember an OOM state
2208 * in the task and mem_cgroup_oom_synchronize() has to be called at
2209 * the end of the page fault to complete the OOM handling.
2210 *
2211 * Returns %true if an ongoing memcg OOM situation was detected and
2212 * completed, %false otherwise.
2213 */
2214bool mem_cgroup_oom_synchronize(bool handle)
2215{
2216 struct mem_cgroup *memcg = current->memcg_oom.memcg;
2217 struct oom_wait_info owait;
2218 bool locked;
2219
2220 /* OOM is global, do not handle */
2221 if (!memcg)
2222 return false;
2223
2224 if (!handle)
2225 goto cleanup;
2226
2227 owait.memcg = memcg;
2228 owait.wait.flags = 0;
2229 owait.wait.func = memcg_oom_wake_function;
2230 owait.wait.private = current;
2231 INIT_LIST_HEAD(&owait.wait.task_list);
2232
2233 prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002234 mem_cgroup_mark_under_oom(memcg);
2235
2236 locked = mem_cgroup_oom_trylock(memcg);
2237
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002238 if (locked)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002239 mem_cgroup_oom_notify(memcg);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002240
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002241 if (locked && !memcg->oom_kill_disable) {
2242 mem_cgroup_unmark_under_oom(memcg);
Johannes Weiner49426422013-10-16 13:46:59 -07002243 finish_wait(&memcg_oom_waitq, &owait.wait);
2244 mem_cgroup_out_of_memory(memcg, current->memcg_oom.gfp_mask,
2245 current->memcg_oom.order);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07002246 } else {
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002247 schedule();
Johannes Weiner49426422013-10-16 13:46:59 -07002248 mem_cgroup_unmark_under_oom(memcg);
2249 finish_wait(&memcg_oom_waitq, &owait.wait);
2250 }
2251
2252 if (locked) {
Johannes Weinerfb2a6fc2013-09-12 15:13:43 -07002253 mem_cgroup_oom_unlock(memcg);
2254 /*
2255 * There is no guarantee that an OOM-lock contender
2256 * sees the wakeups triggered by the OOM kill
2257 * uncharges. Wake any sleepers explicitely.
2258 */
2259 memcg_oom_recover(memcg);
2260 }
Johannes Weiner49426422013-10-16 13:46:59 -07002261cleanup:
2262 current->memcg_oom.memcg = NULL;
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002263 css_put(&memcg->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002264 return true;
KAMEZAWA Hiroyuki0b7f5692009-04-02 16:57:38 -07002265}
2266
Balbir Singhd69b0422009-06-17 16:26:34 -07002267/*
2268 * Currently used to update mapped file statistics, but the routine can be
2269 * generalized to update other statistics as well.
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002270 *
2271 * Notes: Race condition
2272 *
2273 * We usually use page_cgroup_lock() for accessing page_cgroup member but
2274 * it tends to be costly. But considering some conditions, we doesn't need
2275 * to do so _always_.
2276 *
2277 * Considering "charge", lock_page_cgroup() is not required because all
2278 * file-stat operations happen after a page is attached to radix-tree. There
2279 * are no race with "charge".
2280 *
2281 * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
2282 * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
2283 * if there are race with "uncharge". Statistics itself is properly handled
2284 * by flags.
2285 *
2286 * Considering "move", this is an only case we see a race. To make the race
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07002287 * small, we check mm->moving_account and detect there are possibility of race
2288 * If there is, we take a lock.
Balbir Singhd69b0422009-06-17 16:26:34 -07002289 */
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002290
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002291void __mem_cgroup_begin_update_page_stat(struct page *page,
2292 bool *locked, unsigned long *flags)
2293{
2294 struct mem_cgroup *memcg;
2295 struct page_cgroup *pc;
2296
2297 pc = lookup_page_cgroup(page);
2298again:
2299 memcg = pc->mem_cgroup;
2300 if (unlikely(!memcg || !PageCgroupUsed(pc)))
2301 return;
2302 /*
2303 * If this memory cgroup is not under account moving, we don't
Wanpeng Lida92c472012-07-31 16:43:26 -07002304 * need to take move_lock_mem_cgroup(). Because we already hold
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002305 * rcu_read_lock(), any calls to move_account will be delayed until
Andrew Morton13fd1dd92012-03-21 16:34:26 -07002306 * rcu_read_unlock() if mem_cgroup_stolen() == true.
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002307 */
Andrew Morton13fd1dd92012-03-21 16:34:26 -07002308 if (!mem_cgroup_stolen(memcg))
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002309 return;
2310
2311 move_lock_mem_cgroup(memcg, flags);
2312 if (memcg != pc->mem_cgroup || !PageCgroupUsed(pc)) {
2313 move_unlock_mem_cgroup(memcg, flags);
2314 goto again;
2315 }
2316 *locked = true;
2317}
2318
2319void __mem_cgroup_end_update_page_stat(struct page *page, unsigned long *flags)
2320{
2321 struct page_cgroup *pc = lookup_page_cgroup(page);
2322
2323 /*
2324 * It's guaranteed that pc->mem_cgroup never changes while
2325 * lock is held because a routine modifies pc->mem_cgroup
Wanpeng Lida92c472012-07-31 16:43:26 -07002326 * should take move_lock_mem_cgroup().
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002327 */
2328 move_unlock_mem_cgroup(pc->mem_cgroup, flags);
2329}
2330
Greg Thelen2a7106f2011-01-13 15:47:37 -08002331void mem_cgroup_update_page_stat(struct page *page,
Sha Zhengju68b48762013-09-12 15:13:50 -07002332 enum mem_cgroup_stat_index idx, int val)
Balbir Singhd69b0422009-06-17 16:26:34 -07002333{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002334 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07002335 struct page_cgroup *pc = lookup_page_cgroup(page);
KAMEZAWA Hiroyukidbd4ea72011-01-13 15:47:38 -08002336 unsigned long uninitialized_var(flags);
Balbir Singhd69b0422009-06-17 16:26:34 -07002337
Johannes Weinercfa44942012-01-12 17:18:38 -08002338 if (mem_cgroup_disabled())
Balbir Singhd69b0422009-06-17 16:26:34 -07002339 return;
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002340
Sha Zhengju658b72c2013-09-12 15:13:52 -07002341 VM_BUG_ON(!rcu_read_lock_held());
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002342 memcg = pc->mem_cgroup;
2343 if (unlikely(!memcg || !PageCgroupUsed(pc)))
KAMEZAWA Hiroyuki89c06bd2012-03-21 16:34:25 -07002344 return;
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002345
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002346 this_cpu_add(memcg->stat->count[idx], val);
Balbir Singhd69b0422009-06-17 16:26:34 -07002347}
KAMEZAWA Hiroyuki26174ef2010-10-27 15:33:43 -07002348
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002349/*
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002350 * size of first charge trial. "32" comes from vmscan.c's magic value.
2351 * TODO: maybe necessary to use big numbers in big irons.
2352 */
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002353#define CHARGE_BATCH 32U
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002354struct memcg_stock_pcp {
2355 struct mem_cgroup *cached; /* this never be root cgroup */
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002356 unsigned int nr_pages;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002357 struct work_struct work;
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002358 unsigned long flags;
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -07002359#define FLUSHING_CACHED_CHARGE 0
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002360};
2361static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
Michal Hocko9f50fad2011-08-09 11:56:26 +02002362static DEFINE_MUTEX(percpu_charge_mutex);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002363
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002364/**
2365 * consume_stock: Try to consume stocked charge on this cpu.
2366 * @memcg: memcg to consume from.
2367 * @nr_pages: how many pages to charge.
2368 *
2369 * The charges will only happen if @memcg matches the current cpu's memcg
2370 * stock, and at least @nr_pages are available in that stock. Failure to
2371 * service an allocation will refill the stock.
2372 *
2373 * returns true if successful, false otherwise.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002374 */
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002375static bool consume_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002376{
2377 struct memcg_stock_pcp *stock;
2378 bool ret = true;
2379
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002380 if (nr_pages > CHARGE_BATCH)
2381 return false;
2382
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002383 stock = &get_cpu_var(memcg_stock);
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002384 if (memcg == stock->cached && stock->nr_pages >= nr_pages)
2385 stock->nr_pages -= nr_pages;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002386 else /* need to call res_counter_charge */
2387 ret = false;
2388 put_cpu_var(memcg_stock);
2389 return ret;
2390}
2391
2392/*
2393 * Returns stocks cached in percpu to res_counter and reset cached information.
2394 */
2395static void drain_stock(struct memcg_stock_pcp *stock)
2396{
2397 struct mem_cgroup *old = stock->cached;
2398
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002399 if (stock->nr_pages) {
2400 unsigned long bytes = stock->nr_pages * PAGE_SIZE;
2401
2402 res_counter_uncharge(&old->res, bytes);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002403 if (do_swap_account)
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002404 res_counter_uncharge(&old->memsw, bytes);
2405 stock->nr_pages = 0;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002406 }
2407 stock->cached = NULL;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002408}
2409
2410/*
2411 * This must be called under preempt disabled or must be called by
2412 * a thread which is pinned to local cpu.
2413 */
2414static void drain_local_stock(struct work_struct *dummy)
2415{
2416 struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
2417 drain_stock(stock);
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002418 clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002419}
2420
Michal Hockoe4777492013-02-22 16:35:40 -08002421static void __init memcg_stock_init(void)
2422{
2423 int cpu;
2424
2425 for_each_possible_cpu(cpu) {
2426 struct memcg_stock_pcp *stock =
2427 &per_cpu(memcg_stock, cpu);
2428 INIT_WORK(&stock->work, drain_local_stock);
2429 }
2430}
2431
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002432/*
2433 * Cache charges(val) which is from res_counter, to local per_cpu area.
Greg Thelen320cc512010-03-15 15:27:28 +01002434 * This will be consumed by consume_stock() function, later.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002435 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002436static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002437{
2438 struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
2439
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002440 if (stock->cached != memcg) { /* reset if necessary */
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002441 drain_stock(stock);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002442 stock->cached = memcg;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002443 }
Johannes Weiner11c9ea42011-03-23 16:42:34 -07002444 stock->nr_pages += nr_pages;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002445 put_cpu_var(memcg_stock);
2446}
2447
2448/*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002449 * Drains all per-CPU charge caches for given root_memcg resp. subtree
Michal Hockod38144b2011-07-26 16:08:28 -07002450 * of the hierarchy under it. sync flag says whether we should block
2451 * until the work is done.
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002452 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002453static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002454{
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002455 int cpu, curcpu;
Michal Hockod38144b2011-07-26 16:08:28 -07002456
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002457 /* Notify other cpus that system-wide "drain" is running */
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002458 get_online_cpus();
Johannes Weiner5af12d02011-08-25 15:59:07 -07002459 curcpu = get_cpu();
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002460 for_each_online_cpu(cpu) {
2461 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002462 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002463
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002464 memcg = stock->cached;
2465 if (!memcg || !stock->nr_pages)
KAMEZAWA Hiroyuki26fe6162011-06-15 15:08:45 -07002466 continue;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002467 if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
Michal Hocko3e920412011-07-26 16:08:29 -07002468 continue;
Michal Hockod1a05b62011-07-26 16:08:27 -07002469 if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
2470 if (cpu == curcpu)
2471 drain_local_stock(&stock->work);
2472 else
2473 schedule_work_on(cpu, &stock->work);
2474 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002475 }
Johannes Weiner5af12d02011-08-25 15:59:07 -07002476 put_cpu();
Michal Hockod38144b2011-07-26 16:08:28 -07002477
2478 if (!sync)
2479 goto out;
2480
2481 for_each_online_cpu(cpu) {
2482 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
Michal Hocko9f50fad2011-08-09 11:56:26 +02002483 if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
Michal Hockod38144b2011-07-26 16:08:28 -07002484 flush_work(&stock->work);
2485 }
2486out:
Andrew Mortonf894ffa2013-09-12 15:13:35 -07002487 put_online_cpus();
Michal Hockod38144b2011-07-26 16:08:28 -07002488}
2489
2490/*
2491 * Tries to drain stocked charges in other cpus. This function is asynchronous
2492 * and just put a work per cpu for draining localy on each cpu. Caller can
2493 * expects some charges will be back to res_counter later but cannot wait for
2494 * it.
2495 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002496static void drain_all_stock_async(struct mem_cgroup *root_memcg)
Michal Hockod38144b2011-07-26 16:08:28 -07002497{
Michal Hocko9f50fad2011-08-09 11:56:26 +02002498 /*
2499 * If someone calls draining, avoid adding more kworker runs.
2500 */
2501 if (!mutex_trylock(&percpu_charge_mutex))
2502 return;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002503 drain_all_stock(root_memcg, false);
Michal Hocko9f50fad2011-08-09 11:56:26 +02002504 mutex_unlock(&percpu_charge_mutex);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002505}
2506
2507/* This is a synchronous drain interface. */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002508static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002509{
2510 /* called when force_empty is called */
Michal Hocko9f50fad2011-08-09 11:56:26 +02002511 mutex_lock(&percpu_charge_mutex);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002512 drain_all_stock(root_memcg, true);
Michal Hocko9f50fad2011-08-09 11:56:26 +02002513 mutex_unlock(&percpu_charge_mutex);
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002514}
2515
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002516/*
2517 * This function drains percpu counter value from DEAD cpu and
2518 * move it to local cpu. Note that this function can be preempted.
2519 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002520static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002521{
2522 int i;
2523
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002524 spin_lock(&memcg->pcp_counter_lock);
Johannes Weiner61046212012-05-29 15:07:05 -07002525 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002526 long x = per_cpu(memcg->stat->count[i], cpu);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002527
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002528 per_cpu(memcg->stat->count[i], cpu) = 0;
2529 memcg->nocpu_base.count[i] += x;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002530 }
Johannes Weinere9f89742011-03-23 16:42:37 -07002531 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002532 unsigned long x = per_cpu(memcg->stat->events[i], cpu);
Johannes Weinere9f89742011-03-23 16:42:37 -07002533
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002534 per_cpu(memcg->stat->events[i], cpu) = 0;
2535 memcg->nocpu_base.events[i] += x;
Johannes Weinere9f89742011-03-23 16:42:37 -07002536 }
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002537 spin_unlock(&memcg->pcp_counter_lock);
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002538}
2539
Paul Gortmaker0db06282013-06-19 14:53:51 -04002540static int memcg_cpu_hotplug_callback(struct notifier_block *nb,
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002541 unsigned long action,
2542 void *hcpu)
2543{
2544 int cpu = (unsigned long)hcpu;
2545 struct memcg_stock_pcp *stock;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002546 struct mem_cgroup *iter;
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002547
KAMEZAWA Hiroyuki619d0942012-03-21 16:34:23 -07002548 if (action == CPU_ONLINE)
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07002549 return NOTIFY_OK;
KAMEZAWA Hiroyuki1489eba2010-10-27 15:33:42 -07002550
Kirill A. Shutemovd8330492012-04-12 12:49:11 -07002551 if (action != CPU_DEAD && action != CPU_DEAD_FROZEN)
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002552 return NOTIFY_OK;
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002553
Johannes Weiner9f3a0d02012-01-12 17:17:48 -08002554 for_each_mem_cgroup(iter)
KAMEZAWA Hiroyuki711d3d22010-10-27 15:33:42 -07002555 mem_cgroup_drain_pcp_counter(iter, cpu);
2556
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002557 stock = &per_cpu(memcg_stock, cpu);
2558 drain_stock(stock);
2559 return NOTIFY_OK;
2560}
2561
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002562
2563/* See __mem_cgroup_try_charge() for details */
2564enum {
2565 CHARGE_OK, /* success */
2566 CHARGE_RETRY, /* need to retry but retry is not bad */
2567 CHARGE_NOMEM, /* we can't do more. return -ENOMEM */
2568 CHARGE_WOULDBLOCK, /* GFP_WAIT wasn't set and no enough res. */
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002569};
2570
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002571static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
Suleiman Souhlal4c9c5352012-12-18 14:21:41 -08002572 unsigned int nr_pages, unsigned int min_pages,
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002573 bool invoke_oom)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002574{
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002575 unsigned long csize = nr_pages * PAGE_SIZE;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002576 struct mem_cgroup *mem_over_limit;
2577 struct res_counter *fail_res;
2578 unsigned long flags = 0;
2579 int ret;
2580
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002581 ret = res_counter_charge(&memcg->res, csize, &fail_res);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002582
2583 if (likely(!ret)) {
2584 if (!do_swap_account)
2585 return CHARGE_OK;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002586 ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002587 if (likely(!ret))
2588 return CHARGE_OK;
2589
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002590 res_counter_uncharge(&memcg->res, csize);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002591 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
2592 flags |= MEM_CGROUP_RECLAIM_NOSWAP;
2593 } else
2594 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
Johannes Weiner9221edb2011-02-01 15:52:42 -08002595 /*
Johannes Weiner9221edb2011-02-01 15:52:42 -08002596 * Never reclaim on behalf of optional batching, retry with a
2597 * single page instead.
2598 */
Suleiman Souhlal4c9c5352012-12-18 14:21:41 -08002599 if (nr_pages > min_pages)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002600 return CHARGE_RETRY;
2601
2602 if (!(gfp_mask & __GFP_WAIT))
2603 return CHARGE_WOULDBLOCK;
2604
Suleiman Souhlal4c9c5352012-12-18 14:21:41 -08002605 if (gfp_mask & __GFP_NORETRY)
2606 return CHARGE_NOMEM;
2607
Johannes Weiner56600482012-01-12 17:17:59 -08002608 ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002609 if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
Johannes Weiner19942822011-02-01 15:52:43 -08002610 return CHARGE_RETRY;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002611 /*
Johannes Weiner19942822011-02-01 15:52:43 -08002612 * Even though the limit is exceeded at this point, reclaim
2613 * may have been able to free some pages. Retry the charge
2614 * before killing the task.
2615 *
2616 * Only for regular pages, though: huge pages are rather
2617 * unlikely to succeed so close to the limit, and we fall back
2618 * to regular pages anyway in case of failure.
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002619 */
Suleiman Souhlal4c9c5352012-12-18 14:21:41 -08002620 if (nr_pages <= (1 << PAGE_ALLOC_COSTLY_ORDER) && ret)
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002621 return CHARGE_RETRY;
2622
2623 /*
2624 * At task move, charge accounts can be doubly counted. So, it's
2625 * better to wait until the end of task_move if something is going on.
2626 */
2627 if (mem_cgroup_wait_acct_move(mem_over_limit))
2628 return CHARGE_RETRY;
2629
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002630 if (invoke_oom)
2631 mem_cgroup_oom(mem_over_limit, gfp_mask, get_order(csize));
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002632
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002633 return CHARGE_NOMEM;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002634}
2635
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002636/*
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08002637 * __mem_cgroup_try_charge() does
2638 * 1. detect memcg to be charged against from passed *mm and *ptr,
2639 * 2. update res_counter
2640 * 3. call memory reclaim if necessary.
2641 *
2642 * In some special case, if the task is fatal, fatal_signal_pending() or
2643 * has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
2644 * to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
2645 * as possible without any hazards. 2: all pages should have a valid
2646 * pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
2647 * pointer, that is treated as a charge to root_mem_cgroup.
2648 *
2649 * So __mem_cgroup_try_charge() will return
2650 * 0 ... on success, filling *ptr with a valid memcg pointer.
2651 * -ENOMEM ... charge failure because of resource limits.
2652 * -EINTR ... if thread is fatal. *ptr is filled with root_mem_cgroup.
2653 *
2654 * Unlike the exported interface, an "oom" parameter is added. if oom==true,
2655 * the oom-killer can be invoked.
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002656 */
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08002657static int __mem_cgroup_try_charge(struct mm_struct *mm,
Andrea Arcangeliec168512011-01-13 15:46:56 -08002658 gfp_t gfp_mask,
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002659 unsigned int nr_pages,
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002660 struct mem_cgroup **ptr,
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002661 bool oom)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002662{
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002663 unsigned int batch = max(CHARGE_BATCH, nr_pages);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002664 int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002665 struct mem_cgroup *memcg = NULL;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002666 int ret;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002667
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002668 /*
2669 * Unlike gloval-vm's OOM-kill, we're not in memory shortage
2670 * in system level. So, allow to go ahead dying process in addition to
2671 * MEMDIE process.
2672 */
2673 if (unlikely(test_thread_flag(TIF_MEMDIE)
2674 || fatal_signal_pending(current)))
2675 goto bypass;
KAMEZAWA Hiroyukia636b322009-01-07 18:08:08 -08002676
Johannes Weiner49426422013-10-16 13:46:59 -07002677 if (unlikely(task_in_memcg_oom(current)))
2678 goto bypass;
2679
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002680 /*
Hugh Dickins3be91272008-02-07 00:14:19 -08002681 * We always charge the cgroup the mm_struct belongs to.
2682 * The mm_struct's mem_cgroup changes on task migration if the
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002683 * thread group leader migrates. It's possible that mm is not
Johannes Weiner24467ca2012-07-31 16:45:40 -07002684 * set, if so charge the root memcg (happens for pagecache usage).
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002685 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002686 if (!*ptr && !mm)
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08002687 *ptr = root_mem_cgroup;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002688again:
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002689 if (*ptr) { /* css should be a valid one */
2690 memcg = *ptr;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002691 if (mem_cgroup_is_root(memcg))
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002692 goto done;
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002693 if (consume_stock(memcg, nr_pages))
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002694 goto done;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002695 css_get(&memcg->css);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002696 } else {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002697 struct task_struct *p;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08002698
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002699 rcu_read_lock();
2700 p = rcu_dereference(mm->owner);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002701 /*
KAMEZAWA Hiroyukiebb76ce2010-12-29 14:07:11 -08002702 * Because we don't have task_lock(), "p" can exit.
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002703 * In that case, "memcg" can point to root or p can be NULL with
KAMEZAWA Hiroyukiebb76ce2010-12-29 14:07:11 -08002704 * race with swapoff. Then, we have small risk of mis-accouning.
2705 * But such kind of mis-account by race always happens because
2706 * we don't have cgroup_mutex(). It's overkill and we allo that
2707 * small race, here.
2708 * (*) swapoff at el will charge against mm-struct not against
2709 * task-struct. So, mm->owner can be NULL.
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002710 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002711 memcg = mem_cgroup_from_task(p);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08002712 if (!memcg)
2713 memcg = root_mem_cgroup;
2714 if (mem_cgroup_is_root(memcg)) {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002715 rcu_read_unlock();
2716 goto done;
2717 }
Suleiman Souhlala0956d52012-12-18 14:21:36 -08002718 if (consume_stock(memcg, nr_pages)) {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002719 /*
2720 * It seems dagerous to access memcg without css_get().
2721 * But considering how consume_stok works, it's not
2722 * necessary. If consume_stock success, some charges
2723 * from this memcg are cached on this cpu. So, we
2724 * don't need to call css_get()/css_tryget() before
2725 * calling consume_stock().
2726 */
2727 rcu_read_unlock();
2728 goto done;
2729 }
2730 /* after here, we may be blocked. we need to get refcnt */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002731 if (!css_tryget(&memcg->css)) {
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002732 rcu_read_unlock();
2733 goto again;
2734 }
2735 rcu_read_unlock();
2736 }
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002737
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002738 do {
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002739 bool invoke_oom = oom && !nr_oom_retries;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002740
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002741 /* If killed, bypass charge */
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002742 if (fatal_signal_pending(current)) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002743 css_put(&memcg->css);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002744 goto bypass;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002745 }
KAMEZAWA Hiroyukicdec2e42009-12-15 16:47:08 -08002746
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002747 ret = mem_cgroup_do_charge(memcg, gfp_mask, batch,
2748 nr_pages, invoke_oom);
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002749 switch (ret) {
2750 case CHARGE_OK:
2751 break;
2752 case CHARGE_RETRY: /* not in OOM situation but retry */
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002753 batch = nr_pages;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002754 css_put(&memcg->css);
2755 memcg = NULL;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002756 goto again;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002757 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002758 css_put(&memcg->css);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002759 goto nomem;
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002760 case CHARGE_NOMEM: /* OOM routine works */
Johannes Weiner3812c8c2013-09-12 15:13:44 -07002761 if (!oom || invoke_oom) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002762 css_put(&memcg->css);
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002763 goto nomem;
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07002764 }
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002765 nr_oom_retries--;
2766 break;
Balbir Singh66e17072008-02-07 00:13:56 -08002767 }
KAMEZAWA Hiroyuki4b534332010-08-10 18:02:57 -07002768 } while (ret != CHARGE_OK);
2769
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002770 if (batch > nr_pages)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002771 refill_stock(memcg, batch - nr_pages);
2772 css_put(&memcg->css);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07002773done:
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002774 *ptr = memcg;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002775 return 0;
2776nomem:
Johannes Weiner3168ecb2013-10-31 16:34:13 -07002777 if (!(gfp_mask & __GFP_NOFAIL)) {
2778 *ptr = NULL;
2779 return -ENOMEM;
2780 }
KAMEZAWA Hiroyuki867578c2010-03-10 15:22:39 -08002781bypass:
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08002782 *ptr = root_mem_cgroup;
2783 return -EINTR;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002784}
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002785
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002786/*
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002787 * Somemtimes we have to undo a charge we got by try_charge().
2788 * This function is for that and do uncharge, put css's refcnt.
2789 * gotten by try_charge().
2790 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002791static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
Johannes Weinere7018b8d2011-03-23 16:42:33 -07002792 unsigned int nr_pages)
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002793{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002794 if (!mem_cgroup_is_root(memcg)) {
Johannes Weinere7018b8d2011-03-23 16:42:33 -07002795 unsigned long bytes = nr_pages * PAGE_SIZE;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08002796
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002797 res_counter_uncharge(&memcg->res, bytes);
Johannes Weinere7018b8d2011-03-23 16:42:33 -07002798 if (do_swap_account)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002799 res_counter_uncharge(&memcg->memsw, bytes);
Johannes Weinere7018b8d2011-03-23 16:42:33 -07002800 }
Daisuke Nishimuraa3032a22009-12-15 16:47:10 -08002801}
2802
2803/*
KAMEZAWA Hiroyukid01dd172012-05-29 15:07:03 -07002804 * Cancel chrages in this cgroup....doesn't propagate to parent cgroup.
2805 * This is useful when moving usage to parent cgroup.
2806 */
2807static void __mem_cgroup_cancel_local_charge(struct mem_cgroup *memcg,
2808 unsigned int nr_pages)
2809{
2810 unsigned long bytes = nr_pages * PAGE_SIZE;
2811
2812 if (mem_cgroup_is_root(memcg))
2813 return;
2814
2815 res_counter_uncharge_until(&memcg->res, memcg->res.parent, bytes);
2816 if (do_swap_account)
2817 res_counter_uncharge_until(&memcg->memsw,
2818 memcg->memsw.parent, bytes);
2819}
2820
2821/*
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002822 * A helper function to get mem_cgroup from ID. must be called under
Tejun Heoe9316082012-11-05 09:16:58 -08002823 * rcu_read_lock(). The caller is responsible for calling css_tryget if
2824 * the mem_cgroup is used for charging. (dropping refcnt from swap can be
2825 * called against removed memcg.)
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002826 */
2827static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
2828{
2829 struct cgroup_subsys_state *css;
2830
2831 /* ID 0 is unused ID */
2832 if (!id)
2833 return NULL;
2834 css = css_lookup(&mem_cgroup_subsys, id);
2835 if (!css)
2836 return NULL;
Wanpeng Lib2145142012-07-31 16:46:01 -07002837 return mem_cgroup_from_css(css);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002838}
2839
Wu Fengguange42d9d52009-12-16 12:19:59 +01002840struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002841{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002842 struct mem_cgroup *memcg = NULL;
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002843 struct page_cgroup *pc;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002844 unsigned short id;
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002845 swp_entry_t ent;
2846
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002847 VM_BUG_ON(!PageLocked(page));
2848
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002849 pc = lookup_page_cgroup(page);
Daisuke Nishimurac0bd3f62009-04-30 15:08:11 -07002850 lock_page_cgroup(pc);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002851 if (PageCgroupUsed(pc)) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002852 memcg = pc->mem_cgroup;
2853 if (memcg && !css_tryget(&memcg->css))
2854 memcg = NULL;
Wu Fengguange42d9d52009-12-16 12:19:59 +01002855 } else if (PageSwapCache(page)) {
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002856 ent.val = page_private(page);
Bob Liu9fb4b7c2012-01-12 17:18:48 -08002857 id = lookup_swap_cgroup_id(ent);
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002858 rcu_read_lock();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002859 memcg = mem_cgroup_lookup(id);
2860 if (memcg && !css_tryget(&memcg->css))
2861 memcg = NULL;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07002862 rcu_read_unlock();
Daisuke Nishimura3c776e62009-04-02 16:57:43 -07002863 }
Daisuke Nishimurac0bd3f62009-04-30 15:08:11 -07002864 unlock_page_cgroup(pc);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002865 return memcg;
KAMEZAWA Hiroyukib5a84312009-01-07 18:08:35 -08002866}
2867
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002868static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
Johannes Weiner5564e882011-03-23 16:42:29 -07002869 struct page *page,
Johannes Weiner7ec99d62011-03-23 16:42:36 -07002870 unsigned int nr_pages,
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002871 enum charge_type ctype,
2872 bool lrucare)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08002873{
Johannes Weinerce587e62012-04-24 20:22:33 +02002874 struct page_cgroup *pc = lookup_page_cgroup(page);
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002875 struct zone *uninitialized_var(zone);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07002876 struct lruvec *lruvec;
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002877 bool was_on_lru = false;
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07002878 bool anon;
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002879
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002880 lock_page_cgroup(pc);
Johannes Weiner90deb782012-07-31 16:45:47 -07002881 VM_BUG_ON(PageCgroupUsed(pc));
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08002882 /*
2883 * we don't need page_cgroup_lock about tail pages, becase they are not
2884 * accessed by any other context at this point.
2885 */
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002886
2887 /*
2888 * In some cases, SwapCache and FUSE(splice_buf->radixtree), the page
2889 * may already be on some other mem_cgroup's LRU. Take care of it.
2890 */
2891 if (lrucare) {
2892 zone = page_zone(page);
2893 spin_lock_irq(&zone->lru_lock);
2894 if (PageLRU(page)) {
Hugh Dickinsfa9add62012-05-29 15:07:09 -07002895 lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002896 ClearPageLRU(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07002897 del_page_from_lru_list(page, lruvec, page_lru(page));
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002898 was_on_lru = true;
2899 }
2900 }
2901
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002902 pc->mem_cgroup = memcg;
KAMEZAWA Hiroyuki261fb612009-09-23 15:56:33 -07002903 /*
2904 * We access a page_cgroup asynchronously without lock_page_cgroup().
2905 * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
2906 * is accessed after testing USED bit. To make pc->mem_cgroup visible
2907 * before USED bit, we need memory barrier here.
2908 * See mem_cgroup_add_lru_list(), etc.
Andrew Mortonf894ffa2013-09-12 15:13:35 -07002909 */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08002910 smp_wmb();
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07002911 SetPageCgroupUsed(pc);
Hugh Dickins3be91272008-02-07 00:14:19 -08002912
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002913 if (lrucare) {
2914 if (was_on_lru) {
Hugh Dickinsfa9add62012-05-29 15:07:09 -07002915 lruvec = mem_cgroup_zone_lruvec(zone, pc->mem_cgroup);
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002916 VM_BUG_ON(PageLRU(page));
2917 SetPageLRU(page);
Hugh Dickinsfa9add62012-05-29 15:07:09 -07002918 add_page_to_lru_list(page, lruvec, page_lru(page));
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002919 }
2920 spin_unlock_irq(&zone->lru_lock);
2921 }
2922
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -07002923 if (ctype == MEM_CGROUP_CHARGE_TYPE_ANON)
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07002924 anon = true;
2925 else
2926 anon = false;
2927
David Rientjesb070e652013-05-07 16:18:09 -07002928 mem_cgroup_charge_statistics(memcg, page, anon, nr_pages);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07002929 unlock_page_cgroup(pc);
Hugh Dickins9ce70c02012-03-05 14:59:16 -08002930
KAMEZAWA Hiroyuki430e48632010-03-10 15:22:30 -08002931 /*
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -07002932 * "charge_statistics" updated event counter. Then, check it.
2933 * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
2934 * if they exceeds softlimit.
KAMEZAWA Hiroyuki430e48632010-03-10 15:22:30 -08002935 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07002936 memcg_check_events(memcg, page);
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08002937}
2938
Glauber Costa7cf27982012-12-18 14:22:55 -08002939static DEFINE_MUTEX(set_limit_mutex);
2940
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002941#ifdef CONFIG_MEMCG_KMEM
2942static inline bool memcg_can_account_kmem(struct mem_cgroup *memcg)
2943{
2944 return !mem_cgroup_disabled() && !mem_cgroup_is_root(memcg) &&
2945 (memcg->kmem_account_flags & KMEM_ACCOUNTED_MASK);
2946}
2947
Glauber Costa1f458cb2012-12-18 14:22:50 -08002948/*
2949 * This is a bit cumbersome, but it is rarely used and avoids a backpointer
2950 * in the memcg_cache_params struct.
2951 */
2952static struct kmem_cache *memcg_params_to_cache(struct memcg_cache_params *p)
2953{
2954 struct kmem_cache *cachep;
2955
2956 VM_BUG_ON(p->is_root_cache);
2957 cachep = p->root_cache;
2958 return cachep->memcg_params->memcg_caches[memcg_cache_id(p->memcg)];
2959}
2960
Glauber Costa749c5412012-12-18 14:23:01 -08002961#ifdef CONFIG_SLABINFO
Tejun Heo182446d2013-08-08 20:11:24 -04002962static int mem_cgroup_slabinfo_read(struct cgroup_subsys_state *css,
2963 struct cftype *cft, struct seq_file *m)
Glauber Costa749c5412012-12-18 14:23:01 -08002964{
Tejun Heo182446d2013-08-08 20:11:24 -04002965 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Glauber Costa749c5412012-12-18 14:23:01 -08002966 struct memcg_cache_params *params;
2967
2968 if (!memcg_can_account_kmem(memcg))
2969 return -EIO;
2970
2971 print_slabinfo_header(m);
2972
2973 mutex_lock(&memcg->slab_caches_mutex);
2974 list_for_each_entry(params, &memcg->memcg_slab_caches, list)
2975 cache_show(memcg_params_to_cache(params), m);
2976 mutex_unlock(&memcg->slab_caches_mutex);
2977
2978 return 0;
2979}
2980#endif
2981
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002982static int memcg_charge_kmem(struct mem_cgroup *memcg, gfp_t gfp, u64 size)
2983{
2984 struct res_counter *fail_res;
2985 struct mem_cgroup *_memcg;
2986 int ret = 0;
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002987
2988 ret = res_counter_charge(&memcg->kmem, size, &fail_res);
2989 if (ret)
2990 return ret;
2991
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002992 _memcg = memcg;
2993 ret = __mem_cgroup_try_charge(NULL, gfp, size >> PAGE_SHIFT,
Qiang Huangb9921ec2013-11-12 15:07:22 -08002994 &_memcg, oom_gfp_allowed(gfp));
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08002995
2996 if (ret == -EINTR) {
2997 /*
2998 * __mem_cgroup_try_charge() chosed to bypass to root due to
2999 * OOM kill or fatal signal. Since our only options are to
3000 * either fail the allocation or charge it to this cgroup, do
3001 * it as a temporary condition. But we can't fail. From a
3002 * kmem/slab perspective, the cache has already been selected,
3003 * by mem_cgroup_kmem_get_cache(), so it is too late to change
3004 * our minds.
3005 *
3006 * This condition will only trigger if the task entered
3007 * memcg_charge_kmem in a sane state, but was OOM-killed during
3008 * __mem_cgroup_try_charge() above. Tasks that were already
3009 * dying when the allocation triggers should have been already
3010 * directed to the root cgroup in memcontrol.h
3011 */
3012 res_counter_charge_nofail(&memcg->res, size, &fail_res);
3013 if (do_swap_account)
3014 res_counter_charge_nofail(&memcg->memsw, size,
3015 &fail_res);
3016 ret = 0;
3017 } else if (ret)
3018 res_counter_uncharge(&memcg->kmem, size);
3019
3020 return ret;
3021}
3022
3023static void memcg_uncharge_kmem(struct mem_cgroup *memcg, u64 size)
3024{
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003025 res_counter_uncharge(&memcg->res, size);
3026 if (do_swap_account)
3027 res_counter_uncharge(&memcg->memsw, size);
Glauber Costa7de37682012-12-18 14:22:07 -08003028
3029 /* Not down to 0 */
3030 if (res_counter_uncharge(&memcg->kmem, size))
3031 return;
3032
Li Zefan10d5ebf2013-07-08 16:00:33 -07003033 /*
3034 * Releases a reference taken in kmem_cgroup_css_offline in case
3035 * this last uncharge is racing with the offlining code or it is
3036 * outliving the memcg existence.
3037 *
3038 * The memory barrier imposed by test&clear is paired with the
3039 * explicit one in memcg_kmem_mark_dead().
3040 */
Glauber Costa7de37682012-12-18 14:22:07 -08003041 if (memcg_kmem_test_and_clear_dead(memcg))
Li Zefan10d5ebf2013-07-08 16:00:33 -07003042 css_put(&memcg->css);
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003043}
3044
Glauber Costa2633d7a2012-12-18 14:22:34 -08003045void memcg_cache_list_add(struct mem_cgroup *memcg, struct kmem_cache *cachep)
3046{
3047 if (!memcg)
3048 return;
3049
3050 mutex_lock(&memcg->slab_caches_mutex);
3051 list_add(&cachep->memcg_params->list, &memcg->memcg_slab_caches);
3052 mutex_unlock(&memcg->slab_caches_mutex);
3053}
3054
3055/*
3056 * helper for acessing a memcg's index. It will be used as an index in the
3057 * child cache array in kmem_cache, and also to derive its name. This function
3058 * will return -1 when this is not a kmem-limited memcg.
3059 */
3060int memcg_cache_id(struct mem_cgroup *memcg)
3061{
3062 return memcg ? memcg->kmemcg_id : -1;
3063}
3064
Glauber Costa55007d82012-12-18 14:22:38 -08003065/*
3066 * This ends up being protected by the set_limit mutex, during normal
3067 * operation, because that is its main call site.
3068 *
3069 * But when we create a new cache, we can call this as well if its parent
3070 * is kmem-limited. That will have to hold set_limit_mutex as well.
3071 */
3072int memcg_update_cache_sizes(struct mem_cgroup *memcg)
3073{
3074 int num, ret;
3075
3076 num = ida_simple_get(&kmem_limited_groups,
3077 0, MEMCG_CACHES_MAX_SIZE, GFP_KERNEL);
3078 if (num < 0)
3079 return num;
3080 /*
3081 * After this point, kmem_accounted (that we test atomically in
3082 * the beginning of this conditional), is no longer 0. This
3083 * guarantees only one process will set the following boolean
3084 * to true. We don't need test_and_set because we're protected
3085 * by the set_limit_mutex anyway.
3086 */
3087 memcg_kmem_set_activated(memcg);
3088
3089 ret = memcg_update_all_caches(num+1);
3090 if (ret) {
3091 ida_simple_remove(&kmem_limited_groups, num);
3092 memcg_kmem_clear_activated(memcg);
3093 return ret;
3094 }
3095
3096 memcg->kmemcg_id = num;
3097 INIT_LIST_HEAD(&memcg->memcg_slab_caches);
3098 mutex_init(&memcg->slab_caches_mutex);
3099 return 0;
3100}
3101
3102static size_t memcg_caches_array_size(int num_groups)
3103{
3104 ssize_t size;
3105 if (num_groups <= 0)
3106 return 0;
3107
3108 size = 2 * num_groups;
3109 if (size < MEMCG_CACHES_MIN_SIZE)
3110 size = MEMCG_CACHES_MIN_SIZE;
3111 else if (size > MEMCG_CACHES_MAX_SIZE)
3112 size = MEMCG_CACHES_MAX_SIZE;
3113
3114 return size;
3115}
3116
3117/*
3118 * We should update the current array size iff all caches updates succeed. This
3119 * can only be done from the slab side. The slab mutex needs to be held when
3120 * calling this.
3121 */
3122void memcg_update_array_size(int num)
3123{
3124 if (num > memcg_limited_groups_array_size)
3125 memcg_limited_groups_array_size = memcg_caches_array_size(num);
3126}
3127
Konstantin Khlebnikov15cf17d2013-03-08 12:43:36 -08003128static void kmem_cache_destroy_work_func(struct work_struct *w);
3129
Glauber Costa55007d82012-12-18 14:22:38 -08003130int memcg_update_cache_size(struct kmem_cache *s, int num_groups)
3131{
3132 struct memcg_cache_params *cur_params = s->memcg_params;
3133
3134 VM_BUG_ON(s->memcg_params && !s->memcg_params->is_root_cache);
3135
3136 if (num_groups > memcg_limited_groups_array_size) {
3137 int i;
3138 ssize_t size = memcg_caches_array_size(num_groups);
3139
3140 size *= sizeof(void *);
Andrey Vagin90c7a792013-09-11 14:22:18 -07003141 size += offsetof(struct memcg_cache_params, memcg_caches);
Glauber Costa55007d82012-12-18 14:22:38 -08003142
3143 s->memcg_params = kzalloc(size, GFP_KERNEL);
3144 if (!s->memcg_params) {
3145 s->memcg_params = cur_params;
3146 return -ENOMEM;
3147 }
3148
3149 s->memcg_params->is_root_cache = true;
3150
3151 /*
3152 * There is the chance it will be bigger than
3153 * memcg_limited_groups_array_size, if we failed an allocation
3154 * in a cache, in which case all caches updated before it, will
3155 * have a bigger array.
3156 *
3157 * But if that is the case, the data after
3158 * memcg_limited_groups_array_size is certainly unused
3159 */
3160 for (i = 0; i < memcg_limited_groups_array_size; i++) {
3161 if (!cur_params->memcg_caches[i])
3162 continue;
3163 s->memcg_params->memcg_caches[i] =
3164 cur_params->memcg_caches[i];
3165 }
3166
3167 /*
3168 * Ideally, we would wait until all caches succeed, and only
3169 * then free the old one. But this is not worth the extra
3170 * pointer per-cache we'd have to have for this.
3171 *
3172 * It is not a big deal if some caches are left with a size
3173 * bigger than the others. And all updates will reset this
3174 * anyway.
3175 */
3176 kfree(cur_params);
3177 }
3178 return 0;
3179}
3180
Glauber Costa943a4512012-12-18 14:23:03 -08003181int memcg_register_cache(struct mem_cgroup *memcg, struct kmem_cache *s,
3182 struct kmem_cache *root_cache)
Glauber Costa2633d7a2012-12-18 14:22:34 -08003183{
Andrey Vagin90c7a792013-09-11 14:22:18 -07003184 size_t size;
Glauber Costa2633d7a2012-12-18 14:22:34 -08003185
3186 if (!memcg_kmem_enabled())
3187 return 0;
3188
Andrey Vagin90c7a792013-09-11 14:22:18 -07003189 if (!memcg) {
3190 size = offsetof(struct memcg_cache_params, memcg_caches);
Glauber Costa55007d82012-12-18 14:22:38 -08003191 size += memcg_limited_groups_array_size * sizeof(void *);
Andrey Vagin90c7a792013-09-11 14:22:18 -07003192 } else
3193 size = sizeof(struct memcg_cache_params);
Glauber Costa55007d82012-12-18 14:22:38 -08003194
Glauber Costa2633d7a2012-12-18 14:22:34 -08003195 s->memcg_params = kzalloc(size, GFP_KERNEL);
3196 if (!s->memcg_params)
3197 return -ENOMEM;
3198
Glauber Costa943a4512012-12-18 14:23:03 -08003199 if (memcg) {
Glauber Costa2633d7a2012-12-18 14:22:34 -08003200 s->memcg_params->memcg = memcg;
Glauber Costa943a4512012-12-18 14:23:03 -08003201 s->memcg_params->root_cache = root_cache;
Andrey Vagin3e6b11d2013-08-13 16:00:47 -07003202 INIT_WORK(&s->memcg_params->destroy,
3203 kmem_cache_destroy_work_func);
Glauber Costa4ba902b2013-02-12 13:46:22 -08003204 } else
3205 s->memcg_params->is_root_cache = true;
3206
Glauber Costa2633d7a2012-12-18 14:22:34 -08003207 return 0;
3208}
3209
3210void memcg_release_cache(struct kmem_cache *s)
3211{
Glauber Costad7f25f82012-12-18 14:22:40 -08003212 struct kmem_cache *root;
3213 struct mem_cgroup *memcg;
3214 int id;
3215
3216 /*
3217 * This happens, for instance, when a root cache goes away before we
3218 * add any memcg.
3219 */
3220 if (!s->memcg_params)
3221 return;
3222
3223 if (s->memcg_params->is_root_cache)
3224 goto out;
3225
3226 memcg = s->memcg_params->memcg;
3227 id = memcg_cache_id(memcg);
3228
3229 root = s->memcg_params->root_cache;
3230 root->memcg_params->memcg_caches[id] = NULL;
Glauber Costad7f25f82012-12-18 14:22:40 -08003231
3232 mutex_lock(&memcg->slab_caches_mutex);
3233 list_del(&s->memcg_params->list);
3234 mutex_unlock(&memcg->slab_caches_mutex);
3235
Li Zefan20f05312013-07-08 16:00:31 -07003236 css_put(&memcg->css);
Glauber Costad7f25f82012-12-18 14:22:40 -08003237out:
Glauber Costa2633d7a2012-12-18 14:22:34 -08003238 kfree(s->memcg_params);
3239}
3240
Glauber Costa0e9d92f2012-12-18 14:22:42 -08003241/*
3242 * During the creation a new cache, we need to disable our accounting mechanism
3243 * altogether. This is true even if we are not creating, but rather just
3244 * enqueing new caches to be created.
3245 *
3246 * This is because that process will trigger allocations; some visible, like
3247 * explicit kmallocs to auxiliary data structures, name strings and internal
3248 * cache structures; some well concealed, like INIT_WORK() that can allocate
3249 * objects during debug.
3250 *
3251 * If any allocation happens during memcg_kmem_get_cache, we will recurse back
3252 * to it. This may not be a bounded recursion: since the first cache creation
3253 * failed to complete (waiting on the allocation), we'll just try to create the
3254 * cache again, failing at the same point.
3255 *
3256 * memcg_kmem_get_cache is prepared to abort after seeing a positive count of
3257 * memcg_kmem_skip_account. So we enclose anything that might allocate memory
3258 * inside the following two functions.
3259 */
3260static inline void memcg_stop_kmem_account(void)
3261{
3262 VM_BUG_ON(!current->mm);
3263 current->memcg_kmem_skip_account++;
3264}
3265
3266static inline void memcg_resume_kmem_account(void)
3267{
3268 VM_BUG_ON(!current->mm);
3269 current->memcg_kmem_skip_account--;
3270}
3271
Glauber Costa1f458cb2012-12-18 14:22:50 -08003272static void kmem_cache_destroy_work_func(struct work_struct *w)
3273{
3274 struct kmem_cache *cachep;
3275 struct memcg_cache_params *p;
3276
3277 p = container_of(w, struct memcg_cache_params, destroy);
3278
3279 cachep = memcg_params_to_cache(p);
3280
Glauber Costa22933152012-12-18 14:22:59 -08003281 /*
3282 * If we get down to 0 after shrink, we could delete right away.
3283 * However, memcg_release_pages() already puts us back in the workqueue
3284 * in that case. If we proceed deleting, we'll get a dangling
3285 * reference, and removing the object from the workqueue in that case
3286 * is unnecessary complication. We are not a fast path.
3287 *
3288 * Note that this case is fundamentally different from racing with
3289 * shrink_slab(): if memcg_cgroup_destroy_cache() is called in
3290 * kmem_cache_shrink, not only we would be reinserting a dead cache
3291 * into the queue, but doing so from inside the worker racing to
3292 * destroy it.
3293 *
3294 * So if we aren't down to zero, we'll just schedule a worker and try
3295 * again
3296 */
3297 if (atomic_read(&cachep->memcg_params->nr_pages) != 0) {
3298 kmem_cache_shrink(cachep);
3299 if (atomic_read(&cachep->memcg_params->nr_pages) == 0)
3300 return;
3301 } else
Glauber Costa1f458cb2012-12-18 14:22:50 -08003302 kmem_cache_destroy(cachep);
3303}
3304
3305void mem_cgroup_destroy_cache(struct kmem_cache *cachep)
3306{
3307 if (!cachep->memcg_params->dead)
3308 return;
3309
3310 /*
Glauber Costa22933152012-12-18 14:22:59 -08003311 * There are many ways in which we can get here.
3312 *
3313 * We can get to a memory-pressure situation while the delayed work is
3314 * still pending to run. The vmscan shrinkers can then release all
3315 * cache memory and get us to destruction. If this is the case, we'll
3316 * be executed twice, which is a bug (the second time will execute over
3317 * bogus data). In this case, cancelling the work should be fine.
3318 *
3319 * But we can also get here from the worker itself, if
3320 * kmem_cache_shrink is enough to shake all the remaining objects and
3321 * get the page count to 0. In this case, we'll deadlock if we try to
3322 * cancel the work (the worker runs with an internal lock held, which
3323 * is the same lock we would hold for cancel_work_sync().)
3324 *
3325 * Since we can't possibly know who got us here, just refrain from
3326 * running if there is already work pending
3327 */
3328 if (work_pending(&cachep->memcg_params->destroy))
3329 return;
3330 /*
Glauber Costa1f458cb2012-12-18 14:22:50 -08003331 * We have to defer the actual destroying to a workqueue, because
3332 * we might currently be in a context that cannot sleep.
3333 */
3334 schedule_work(&cachep->memcg_params->destroy);
3335}
3336
Glauber Costad7f25f82012-12-18 14:22:40 -08003337/*
3338 * This lock protects updaters, not readers. We want readers to be as fast as
3339 * they can, and they will either see NULL or a valid cache value. Our model
3340 * allow them to see NULL, in which case the root memcg will be selected.
3341 *
3342 * We need this lock because multiple allocations to the same cache from a non
3343 * will span more than one worker. Only one of them can create the cache.
3344 */
3345static DEFINE_MUTEX(memcg_cache_mutex);
Michal Hockod9c10dd2013-03-28 08:48:14 +01003346
3347/*
3348 * Called with memcg_cache_mutex held
3349 */
3350static struct kmem_cache *kmem_cache_dup(struct mem_cgroup *memcg,
3351 struct kmem_cache *s)
3352{
3353 struct kmem_cache *new;
3354 static char *tmp_name = NULL;
3355
3356 lockdep_assert_held(&memcg_cache_mutex);
3357
3358 /*
3359 * kmem_cache_create_memcg duplicates the given name and
3360 * cgroup_name for this name requires RCU context.
3361 * This static temporary buffer is used to prevent from
3362 * pointless shortliving allocation.
3363 */
3364 if (!tmp_name) {
3365 tmp_name = kmalloc(PATH_MAX, GFP_KERNEL);
3366 if (!tmp_name)
3367 return NULL;
3368 }
3369
3370 rcu_read_lock();
3371 snprintf(tmp_name, PATH_MAX, "%s(%d:%s)", s->name,
3372 memcg_cache_id(memcg), cgroup_name(memcg->css.cgroup));
3373 rcu_read_unlock();
3374
3375 new = kmem_cache_create_memcg(memcg, tmp_name, s->object_size, s->align,
3376 (s->flags & ~SLAB_PANIC), s->ctor, s);
3377
3378 if (new)
3379 new->allocflags |= __GFP_KMEMCG;
3380
3381 return new;
3382}
3383
Glauber Costad7f25f82012-12-18 14:22:40 -08003384static struct kmem_cache *memcg_create_kmem_cache(struct mem_cgroup *memcg,
3385 struct kmem_cache *cachep)
3386{
3387 struct kmem_cache *new_cachep;
3388 int idx;
3389
3390 BUG_ON(!memcg_can_account_kmem(memcg));
3391
3392 idx = memcg_cache_id(memcg);
3393
3394 mutex_lock(&memcg_cache_mutex);
3395 new_cachep = cachep->memcg_params->memcg_caches[idx];
Li Zefan20f05312013-07-08 16:00:31 -07003396 if (new_cachep) {
3397 css_put(&memcg->css);
Glauber Costad7f25f82012-12-18 14:22:40 -08003398 goto out;
Li Zefan20f05312013-07-08 16:00:31 -07003399 }
Glauber Costad7f25f82012-12-18 14:22:40 -08003400
3401 new_cachep = kmem_cache_dup(memcg, cachep);
Glauber Costad7f25f82012-12-18 14:22:40 -08003402 if (new_cachep == NULL) {
3403 new_cachep = cachep;
Li Zefan20f05312013-07-08 16:00:31 -07003404 css_put(&memcg->css);
Glauber Costad7f25f82012-12-18 14:22:40 -08003405 goto out;
3406 }
3407
Glauber Costa1f458cb2012-12-18 14:22:50 -08003408 atomic_set(&new_cachep->memcg_params->nr_pages , 0);
Glauber Costad7f25f82012-12-18 14:22:40 -08003409
3410 cachep->memcg_params->memcg_caches[idx] = new_cachep;
3411 /*
3412 * the readers won't lock, make sure everybody sees the updated value,
3413 * so they won't put stuff in the queue again for no reason
3414 */
3415 wmb();
3416out:
3417 mutex_unlock(&memcg_cache_mutex);
3418 return new_cachep;
3419}
3420
Glauber Costa7cf27982012-12-18 14:22:55 -08003421void kmem_cache_destroy_memcg_children(struct kmem_cache *s)
3422{
3423 struct kmem_cache *c;
3424 int i;
3425
3426 if (!s->memcg_params)
3427 return;
3428 if (!s->memcg_params->is_root_cache)
3429 return;
3430
3431 /*
3432 * If the cache is being destroyed, we trust that there is no one else
3433 * requesting objects from it. Even if there are, the sanity checks in
3434 * kmem_cache_destroy should caught this ill-case.
3435 *
3436 * Still, we don't want anyone else freeing memcg_caches under our
3437 * noses, which can happen if a new memcg comes to life. As usual,
3438 * we'll take the set_limit_mutex to protect ourselves against this.
3439 */
3440 mutex_lock(&set_limit_mutex);
3441 for (i = 0; i < memcg_limited_groups_array_size; i++) {
3442 c = s->memcg_params->memcg_caches[i];
3443 if (!c)
3444 continue;
3445
3446 /*
3447 * We will now manually delete the caches, so to avoid races
3448 * we need to cancel all pending destruction workers and
3449 * proceed with destruction ourselves.
3450 *
3451 * kmem_cache_destroy() will call kmem_cache_shrink internally,
3452 * and that could spawn the workers again: it is likely that
3453 * the cache still have active pages until this very moment.
3454 * This would lead us back to mem_cgroup_destroy_cache.
3455 *
3456 * But that will not execute at all if the "dead" flag is not
3457 * set, so flip it down to guarantee we are in control.
3458 */
3459 c->memcg_params->dead = false;
Glauber Costa22933152012-12-18 14:22:59 -08003460 cancel_work_sync(&c->memcg_params->destroy);
Glauber Costa7cf27982012-12-18 14:22:55 -08003461 kmem_cache_destroy(c);
3462 }
3463 mutex_unlock(&set_limit_mutex);
3464}
3465
Glauber Costad7f25f82012-12-18 14:22:40 -08003466struct create_work {
3467 struct mem_cgroup *memcg;
3468 struct kmem_cache *cachep;
3469 struct work_struct work;
3470};
3471
Glauber Costa1f458cb2012-12-18 14:22:50 -08003472static void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
3473{
3474 struct kmem_cache *cachep;
3475 struct memcg_cache_params *params;
3476
3477 if (!memcg_kmem_is_active(memcg))
3478 return;
3479
3480 mutex_lock(&memcg->slab_caches_mutex);
3481 list_for_each_entry(params, &memcg->memcg_slab_caches, list) {
3482 cachep = memcg_params_to_cache(params);
3483 cachep->memcg_params->dead = true;
Glauber Costa1f458cb2012-12-18 14:22:50 -08003484 schedule_work(&cachep->memcg_params->destroy);
3485 }
3486 mutex_unlock(&memcg->slab_caches_mutex);
3487}
3488
Glauber Costad7f25f82012-12-18 14:22:40 -08003489static void memcg_create_cache_work_func(struct work_struct *w)
3490{
3491 struct create_work *cw;
3492
3493 cw = container_of(w, struct create_work, work);
3494 memcg_create_kmem_cache(cw->memcg, cw->cachep);
Glauber Costad7f25f82012-12-18 14:22:40 -08003495 kfree(cw);
3496}
3497
3498/*
3499 * Enqueue the creation of a per-memcg kmem_cache.
Glauber Costad7f25f82012-12-18 14:22:40 -08003500 */
Glauber Costa0e9d92f2012-12-18 14:22:42 -08003501static void __memcg_create_cache_enqueue(struct mem_cgroup *memcg,
3502 struct kmem_cache *cachep)
Glauber Costad7f25f82012-12-18 14:22:40 -08003503{
3504 struct create_work *cw;
3505
3506 cw = kmalloc(sizeof(struct create_work), GFP_NOWAIT);
Li Zefanca0dde92013-04-29 15:08:57 -07003507 if (cw == NULL) {
3508 css_put(&memcg->css);
Glauber Costad7f25f82012-12-18 14:22:40 -08003509 return;
3510 }
3511
3512 cw->memcg = memcg;
3513 cw->cachep = cachep;
3514
3515 INIT_WORK(&cw->work, memcg_create_cache_work_func);
3516 schedule_work(&cw->work);
3517}
3518
Glauber Costa0e9d92f2012-12-18 14:22:42 -08003519static void memcg_create_cache_enqueue(struct mem_cgroup *memcg,
3520 struct kmem_cache *cachep)
3521{
3522 /*
3523 * We need to stop accounting when we kmalloc, because if the
3524 * corresponding kmalloc cache is not yet created, the first allocation
3525 * in __memcg_create_cache_enqueue will recurse.
3526 *
3527 * However, it is better to enclose the whole function. Depending on
3528 * the debugging options enabled, INIT_WORK(), for instance, can
3529 * trigger an allocation. This too, will make us recurse. Because at
3530 * this point we can't allow ourselves back into memcg_kmem_get_cache,
3531 * the safest choice is to do it like this, wrapping the whole function.
3532 */
3533 memcg_stop_kmem_account();
3534 __memcg_create_cache_enqueue(memcg, cachep);
3535 memcg_resume_kmem_account();
3536}
Glauber Costad7f25f82012-12-18 14:22:40 -08003537/*
3538 * Return the kmem_cache we're supposed to use for a slab allocation.
3539 * We try to use the current memcg's version of the cache.
3540 *
3541 * If the cache does not exist yet, if we are the first user of it,
3542 * we either create it immediately, if possible, or create it asynchronously
3543 * in a workqueue.
3544 * In the latter case, we will let the current allocation go through with
3545 * the original cache.
3546 *
3547 * Can't be called in interrupt context or from kernel threads.
3548 * This function needs to be called with rcu_read_lock() held.
3549 */
3550struct kmem_cache *__memcg_kmem_get_cache(struct kmem_cache *cachep,
3551 gfp_t gfp)
3552{
3553 struct mem_cgroup *memcg;
3554 int idx;
3555
3556 VM_BUG_ON(!cachep->memcg_params);
3557 VM_BUG_ON(!cachep->memcg_params->is_root_cache);
3558
Glauber Costa0e9d92f2012-12-18 14:22:42 -08003559 if (!current->mm || current->memcg_kmem_skip_account)
3560 return cachep;
3561
Glauber Costad7f25f82012-12-18 14:22:40 -08003562 rcu_read_lock();
3563 memcg = mem_cgroup_from_task(rcu_dereference(current->mm->owner));
Glauber Costad7f25f82012-12-18 14:22:40 -08003564
3565 if (!memcg_can_account_kmem(memcg))
Li Zefanca0dde92013-04-29 15:08:57 -07003566 goto out;
Glauber Costad7f25f82012-12-18 14:22:40 -08003567
3568 idx = memcg_cache_id(memcg);
3569
3570 /*
3571 * barrier to mare sure we're always seeing the up to date value. The
3572 * code updating memcg_caches will issue a write barrier to match this.
3573 */
3574 read_barrier_depends();
Li Zefanca0dde92013-04-29 15:08:57 -07003575 if (likely(cachep->memcg_params->memcg_caches[idx])) {
3576 cachep = cachep->memcg_params->memcg_caches[idx];
3577 goto out;
Glauber Costad7f25f82012-12-18 14:22:40 -08003578 }
3579
Li Zefanca0dde92013-04-29 15:08:57 -07003580 /* The corresponding put will be done in the workqueue. */
3581 if (!css_tryget(&memcg->css))
3582 goto out;
3583 rcu_read_unlock();
3584
3585 /*
3586 * If we are in a safe context (can wait, and not in interrupt
3587 * context), we could be be predictable and return right away.
3588 * This would guarantee that the allocation being performed
3589 * already belongs in the new cache.
3590 *
3591 * However, there are some clashes that can arrive from locking.
3592 * For instance, because we acquire the slab_mutex while doing
3593 * kmem_cache_dup, this means no further allocation could happen
3594 * with the slab_mutex held.
3595 *
3596 * Also, because cache creation issue get_online_cpus(), this
3597 * creates a lock chain: memcg_slab_mutex -> cpu_hotplug_mutex,
3598 * that ends up reversed during cpu hotplug. (cpuset allocates
3599 * a bunch of GFP_KERNEL memory during cpuup). Due to all that,
3600 * better to defer everything.
3601 */
3602 memcg_create_cache_enqueue(memcg, cachep);
3603 return cachep;
3604out:
3605 rcu_read_unlock();
3606 return cachep;
Glauber Costad7f25f82012-12-18 14:22:40 -08003607}
3608EXPORT_SYMBOL(__memcg_kmem_get_cache);
3609
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003610/*
3611 * We need to verify if the allocation against current->mm->owner's memcg is
3612 * possible for the given order. But the page is not allocated yet, so we'll
3613 * need a further commit step to do the final arrangements.
3614 *
3615 * It is possible for the task to switch cgroups in this mean time, so at
3616 * commit time, we can't rely on task conversion any longer. We'll then use
3617 * the handle argument to return to the caller which cgroup we should commit
3618 * against. We could also return the memcg directly and avoid the pointer
3619 * passing, but a boolean return value gives better semantics considering
3620 * the compiled-out case as well.
3621 *
3622 * Returning true means the allocation is possible.
3623 */
3624bool
3625__memcg_kmem_newpage_charge(gfp_t gfp, struct mem_cgroup **_memcg, int order)
3626{
3627 struct mem_cgroup *memcg;
3628 int ret;
3629
3630 *_memcg = NULL;
Glauber Costa6d42c232013-07-08 16:00:00 -07003631
3632 /*
3633 * Disabling accounting is only relevant for some specific memcg
3634 * internal allocations. Therefore we would initially not have such
3635 * check here, since direct calls to the page allocator that are marked
3636 * with GFP_KMEMCG only happen outside memcg core. We are mostly
3637 * concerned with cache allocations, and by having this test at
3638 * memcg_kmem_get_cache, we are already able to relay the allocation to
3639 * the root cache and bypass the memcg cache altogether.
3640 *
3641 * There is one exception, though: the SLUB allocator does not create
3642 * large order caches, but rather service large kmallocs directly from
3643 * the page allocator. Therefore, the following sequence when backed by
3644 * the SLUB allocator:
3645 *
Andrew Mortonf894ffa2013-09-12 15:13:35 -07003646 * memcg_stop_kmem_account();
3647 * kmalloc(<large_number>)
3648 * memcg_resume_kmem_account();
Glauber Costa6d42c232013-07-08 16:00:00 -07003649 *
3650 * would effectively ignore the fact that we should skip accounting,
3651 * since it will drive us directly to this function without passing
3652 * through the cache selector memcg_kmem_get_cache. Such large
3653 * allocations are extremely rare but can happen, for instance, for the
3654 * cache arrays. We bring this test here.
3655 */
3656 if (!current->mm || current->memcg_kmem_skip_account)
3657 return true;
3658
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003659 memcg = try_get_mem_cgroup_from_mm(current->mm);
3660
3661 /*
3662 * very rare case described in mem_cgroup_from_task. Unfortunately there
3663 * isn't much we can do without complicating this too much, and it would
3664 * be gfp-dependent anyway. Just let it go
3665 */
3666 if (unlikely(!memcg))
3667 return true;
3668
3669 if (!memcg_can_account_kmem(memcg)) {
3670 css_put(&memcg->css);
3671 return true;
3672 }
3673
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003674 ret = memcg_charge_kmem(memcg, gfp, PAGE_SIZE << order);
3675 if (!ret)
3676 *_memcg = memcg;
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003677
3678 css_put(&memcg->css);
3679 return (ret == 0);
3680}
3681
3682void __memcg_kmem_commit_charge(struct page *page, struct mem_cgroup *memcg,
3683 int order)
3684{
3685 struct page_cgroup *pc;
3686
3687 VM_BUG_ON(mem_cgroup_is_root(memcg));
3688
3689 /* The page allocation failed. Revert */
3690 if (!page) {
3691 memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003692 return;
3693 }
3694
3695 pc = lookup_page_cgroup(page);
3696 lock_page_cgroup(pc);
3697 pc->mem_cgroup = memcg;
3698 SetPageCgroupUsed(pc);
3699 unlock_page_cgroup(pc);
3700}
3701
3702void __memcg_kmem_uncharge_pages(struct page *page, int order)
3703{
3704 struct mem_cgroup *memcg = NULL;
3705 struct page_cgroup *pc;
3706
3707
3708 pc = lookup_page_cgroup(page);
3709 /*
3710 * Fast unlocked return. Theoretically might have changed, have to
3711 * check again after locking.
3712 */
3713 if (!PageCgroupUsed(pc))
3714 return;
3715
3716 lock_page_cgroup(pc);
3717 if (PageCgroupUsed(pc)) {
3718 memcg = pc->mem_cgroup;
3719 ClearPageCgroupUsed(pc);
3720 }
3721 unlock_page_cgroup(pc);
3722
3723 /*
3724 * We trust that only if there is a memcg associated with the page, it
3725 * is a valid allocation
3726 */
3727 if (!memcg)
3728 return;
3729
3730 VM_BUG_ON(mem_cgroup_is_root(memcg));
3731 memcg_uncharge_kmem(memcg, PAGE_SIZE << order);
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003732}
Glauber Costa1f458cb2012-12-18 14:22:50 -08003733#else
3734static inline void mem_cgroup_destroy_all_caches(struct mem_cgroup *memcg)
3735{
3736}
Glauber Costa7ae1e1d2012-12-18 14:21:56 -08003737#endif /* CONFIG_MEMCG_KMEM */
3738
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003739#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3740
Kirill A. Shutemova0db00f2012-05-29 15:06:56 -07003741#define PCGF_NOCOPY_AT_SPLIT (1 << PCG_LOCK | 1 << PCG_MIGRATION)
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003742/*
3743 * Because tail pages are not marked as "used", set it. We're under
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003744 * zone->lru_lock, 'splitting on pmd' and compound_lock.
3745 * charge/uncharge will be never happen and move_account() is done under
3746 * compound_lock(), so we don't have to take care of races.
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003747 */
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003748void mem_cgroup_split_huge_fixup(struct page *head)
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003749{
3750 struct page_cgroup *head_pc = lookup_page_cgroup(head);
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003751 struct page_cgroup *pc;
David Rientjesb070e652013-05-07 16:18:09 -07003752 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003753 int i;
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003754
KAMEZAWA Hiroyuki3d37c4a2011-01-25 15:07:28 -08003755 if (mem_cgroup_disabled())
3756 return;
David Rientjesb070e652013-05-07 16:18:09 -07003757
3758 memcg = head_pc->mem_cgroup;
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003759 for (i = 1; i < HPAGE_PMD_NR; i++) {
3760 pc = head_pc + i;
David Rientjesb070e652013-05-07 16:18:09 -07003761 pc->mem_cgroup = memcg;
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003762 smp_wmb();/* see __commit_charge() */
KAMEZAWA Hiroyukie94c8a92012-01-12 17:18:20 -08003763 pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
3764 }
David Rientjesb070e652013-05-07 16:18:09 -07003765 __this_cpu_sub(memcg->stat->count[MEM_CGROUP_STAT_RSS_HUGE],
3766 HPAGE_PMD_NR);
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003767}
Hugh Dickins12d27102012-01-12 17:19:52 -08003768#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
KAMEZAWA Hiroyukica3e0212011-01-20 14:44:24 -08003769
Sha Zhengju3ea67d02013-09-12 15:13:53 -07003770static inline
3771void mem_cgroup_move_account_page_stat(struct mem_cgroup *from,
3772 struct mem_cgroup *to,
3773 unsigned int nr_pages,
3774 enum mem_cgroup_stat_index idx)
3775{
3776 /* Update stat data for mem_cgroup */
3777 preempt_disable();
Greg Thelen5e8cfc32013-10-30 13:56:21 -07003778 __this_cpu_sub(from->stat->count[idx], nr_pages);
Sha Zhengju3ea67d02013-09-12 15:13:53 -07003779 __this_cpu_add(to->stat->count[idx], nr_pages);
3780 preempt_enable();
3781}
3782
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003783/**
Johannes Weinerde3638d2011-03-23 16:42:28 -07003784 * mem_cgroup_move_account - move account of the page
Johannes Weiner5564e882011-03-23 16:42:29 -07003785 * @page: the page
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003786 * @nr_pages: number of regular pages (>1 for huge pages)
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003787 * @pc: page_cgroup of the page.
3788 * @from: mem_cgroup which the page is moved from.
3789 * @to: mem_cgroup which the page is moved to. @from != @to.
3790 *
3791 * The caller must confirm following.
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003792 * - page is not on LRU (isolate_page() is useful.)
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003793 * - compound_lock is held when nr_pages > 1
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003794 *
KAMEZAWA Hiroyuki2f3479b2012-05-29 15:07:04 -07003795 * This function doesn't do "charge" to new cgroup and doesn't do "uncharge"
3796 * from old cgroup.
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003797 */
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003798static int mem_cgroup_move_account(struct page *page,
3799 unsigned int nr_pages,
3800 struct page_cgroup *pc,
3801 struct mem_cgroup *from,
KAMEZAWA Hiroyuki2f3479b2012-05-29 15:07:04 -07003802 struct mem_cgroup *to)
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003803{
Johannes Weinerde3638d2011-03-23 16:42:28 -07003804 unsigned long flags;
3805 int ret;
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07003806 bool anon = PageAnon(page);
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08003807
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003808 VM_BUG_ON(from == to);
Johannes Weiner5564e882011-03-23 16:42:29 -07003809 VM_BUG_ON(PageLRU(page));
Johannes Weinerde3638d2011-03-23 16:42:28 -07003810 /*
3811 * The page is isolated from LRU. So, collapse function
3812 * will not handle this page. But page splitting can happen.
3813 * Do this check under compound_page_lock(). The caller should
3814 * hold it.
3815 */
3816 ret = -EBUSY;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003817 if (nr_pages > 1 && !PageTransHuge(page))
Johannes Weinerde3638d2011-03-23 16:42:28 -07003818 goto out;
3819
3820 lock_page_cgroup(pc);
3821
3822 ret = -EINVAL;
3823 if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
3824 goto unlock;
3825
KAMEZAWA Hiroyuki312734c02012-03-21 16:34:24 -07003826 move_lock_mem_cgroup(from, &flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003827
Sha Zhengju3ea67d02013-09-12 15:13:53 -07003828 if (!anon && page_mapped(page))
3829 mem_cgroup_move_account_page_stat(from, to, nr_pages,
3830 MEM_CGROUP_STAT_FILE_MAPPED);
3831
3832 if (PageWriteback(page))
3833 mem_cgroup_move_account_page_stat(from, to, nr_pages,
3834 MEM_CGROUP_STAT_WRITEBACK);
3835
David Rientjesb070e652013-05-07 16:18:09 -07003836 mem_cgroup_charge_statistics(from, page, anon, -nr_pages);
Balbir Singhd69b0422009-06-17 16:26:34 -07003837
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08003838 /* caller should have done css_get */
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003839 pc->mem_cgroup = to;
David Rientjesb070e652013-05-07 16:18:09 -07003840 mem_cgroup_charge_statistics(to, page, anon, nr_pages);
KAMEZAWA Hiroyuki312734c02012-03-21 16:34:24 -07003841 move_unlock_mem_cgroup(from, &flags);
Johannes Weinerde3638d2011-03-23 16:42:28 -07003842 ret = 0;
3843unlock:
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08003844 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukid2265e62010-03-10 15:22:31 -08003845 /*
3846 * check events
3847 */
Johannes Weiner5564e882011-03-23 16:42:29 -07003848 memcg_check_events(to, page);
3849 memcg_check_events(from, page);
Johannes Weinerde3638d2011-03-23 16:42:28 -07003850out:
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003851 return ret;
3852}
3853
Michal Hocko2ef37d32012-10-26 13:37:30 +02003854/**
3855 * mem_cgroup_move_parent - moves page to the parent group
3856 * @page: the page to move
3857 * @pc: page_cgroup of the page
3858 * @child: page's cgroup
3859 *
3860 * move charges to its parent or the root cgroup if the group has no
3861 * parent (aka use_hierarchy==0).
3862 * Although this might fail (get_page_unless_zero, isolate_lru_page or
3863 * mem_cgroup_move_account fails) the failure is always temporary and
3864 * it signals a race with a page removal/uncharge or migration. In the
3865 * first case the page is on the way out and it will vanish from the LRU
3866 * on the next attempt and the call should be retried later.
3867 * Isolation from the LRU fails only if page has been isolated from
3868 * the LRU since we looked at it and that usually means either global
3869 * reclaim or migration going on. The page will either get back to the
3870 * LRU or vanish.
3871 * Finaly mem_cgroup_move_account fails only if the page got uncharged
3872 * (!PageCgroupUsed) or moved to a different group. The page will
3873 * disappear in the next attempt.
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003874 */
Johannes Weiner5564e882011-03-23 16:42:29 -07003875static int mem_cgroup_move_parent(struct page *page,
3876 struct page_cgroup *pc,
KAMEZAWA Hiroyuki6068bf02012-07-31 16:42:45 -07003877 struct mem_cgroup *child)
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003878{
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003879 struct mem_cgroup *parent;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003880 unsigned int nr_pages;
Andrew Morton4be44892011-03-23 16:42:39 -07003881 unsigned long uninitialized_var(flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003882 int ret;
3883
Michal Hockod8423012012-10-26 13:37:29 +02003884 VM_BUG_ON(mem_cgroup_is_root(child));
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003885
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08003886 ret = -EBUSY;
3887 if (!get_page_unless_zero(page))
3888 goto out;
3889 if (isolate_lru_page(page))
3890 goto put;
KAMEZAWA Hiroyuki52dbb902011-01-25 15:07:29 -08003891
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003892 nr_pages = hpage_nr_pages(page);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003893
KAMEZAWA Hiroyukicc926f72012-05-29 15:07:04 -07003894 parent = parent_mem_cgroup(child);
3895 /*
3896 * If no parent, move charges to root cgroup.
3897 */
3898 if (!parent)
3899 parent = root_mem_cgroup;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003900
Michal Hocko2ef37d32012-10-26 13:37:30 +02003901 if (nr_pages > 1) {
3902 VM_BUG_ON(!PageTransHuge(page));
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08003903 flags = compound_lock_irqsave(page);
Michal Hocko2ef37d32012-10-26 13:37:30 +02003904 }
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08003905
KAMEZAWA Hiroyukicc926f72012-05-29 15:07:04 -07003906 ret = mem_cgroup_move_account(page, nr_pages,
KAMEZAWA Hiroyuki2f3479b2012-05-29 15:07:04 -07003907 pc, child, parent);
KAMEZAWA Hiroyukicc926f72012-05-29 15:07:04 -07003908 if (!ret)
3909 __mem_cgroup_cancel_local_charge(child, nr_pages);
Jesper Juhl8dba4742011-01-25 15:07:24 -08003910
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003911 if (nr_pages > 1)
KAMEZAWA Hiroyuki987eba62011-01-20 14:44:25 -08003912 compound_unlock_irqrestore(page, flags);
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08003913 putback_lru_page(page);
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08003914put:
Daisuke Nishimura40d58132009-01-15 13:51:12 -08003915 put_page(page);
Daisuke Nishimura57f9fd7d2009-12-15 16:47:11 -08003916out:
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08003917 return ret;
3918}
3919
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003920/*
3921 * Charge the memory controller for page usage.
3922 * Return
3923 * 0 if the charge was successful
3924 * < 0 if the cgroup is over its limit
3925 */
3926static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
Daisuke Nishimura73045c42010-08-10 18:02:59 -07003927 gfp_t gfp_mask, enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003928{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003929 struct mem_cgroup *memcg = NULL;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003930 unsigned int nr_pages = 1;
Johannes Weiner8493ae42011-02-01 15:52:44 -08003931 bool oom = true;
3932 int ret;
Andrea Arcangeliec168512011-01-13 15:46:56 -08003933
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08003934 if (PageTransHuge(page)) {
Johannes Weiner7ec99d62011-03-23 16:42:36 -07003935 nr_pages <<= compound_order(page);
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08003936 VM_BUG_ON(!PageTransHuge(page));
Johannes Weiner8493ae42011-02-01 15:52:44 -08003937 /*
3938 * Never OOM-kill a process for a huge page. The
3939 * fault handler will fall back to regular pages.
3940 */
3941 oom = false;
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08003942 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003943
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003944 ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08003945 if (ret == -ENOMEM)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003946 return ret;
Johannes Weinerce587e62012-04-24 20:22:33 +02003947 __mem_cgroup_commit_charge(memcg, page, nr_pages, ctype, false);
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08003948 return 0;
3949}
3950
3951int mem_cgroup_newpage_charge(struct page *page,
3952 struct mm_struct *mm, gfp_t gfp_mask)
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08003953{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08003954 if (mem_cgroup_disabled())
Li Zefancede86a2008-07-25 01:47:18 -07003955 return 0;
Johannes Weiner7a0524c2012-01-12 17:18:43 -08003956 VM_BUG_ON(page_mapped(page));
3957 VM_BUG_ON(page->mapping && !PageAnon(page));
3958 VM_BUG_ON(!mm);
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08003959 return mem_cgroup_charge_common(page, mm, gfp_mask,
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -07003960 MEM_CGROUP_CHARGE_TYPE_ANON);
KAMEZAWA Hiroyuki217bc312008-02-07 00:14:17 -08003961}
3962
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08003963/*
3964 * While swap-in, try_charge -> commit or cancel, the page is locked.
3965 * And when try_charge() successfully returns, one refcnt to memcg without
Uwe Kleine-König21ae2952009-10-07 15:21:09 +02003966 * struct page_cgroup is acquired. This refcnt will be consumed by
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08003967 * "commit()" or removed by "cancel()"
3968 */
Johannes Weiner0435a2f2012-07-31 16:45:43 -07003969static int __mem_cgroup_try_charge_swapin(struct mm_struct *mm,
3970 struct page *page,
3971 gfp_t mask,
3972 struct mem_cgroup **memcgp)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003973{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003974 struct mem_cgroup *memcg;
Johannes Weiner90deb782012-07-31 16:45:47 -07003975 struct page_cgroup *pc;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08003976 int ret;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003977
Johannes Weiner90deb782012-07-31 16:45:47 -07003978 pc = lookup_page_cgroup(page);
3979 /*
3980 * Every swap fault against a single page tries to charge the
3981 * page, bail as early as possible. shmem_unuse() encounters
3982 * already charged pages, too. The USED bit is protected by
3983 * the page lock, which serializes swap cache removal, which
3984 * in turn serializes uncharging.
3985 */
3986 if (PageCgroupUsed(pc))
3987 return 0;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003988 if (!do_swap_account)
3989 goto charge_cur_mm;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003990 memcg = try_get_mem_cgroup_from_page(page);
3991 if (!memcg)
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08003992 goto charge_cur_mm;
Johannes Weiner72835c82012-01-12 17:18:32 -08003993 *memcgp = memcg;
3994 ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07003995 css_put(&memcg->css);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08003996 if (ret == -EINTR)
3997 ret = 0;
KAMEZAWA Hiroyuki54595fe2009-01-07 18:08:33 -08003998 return ret;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08003999charge_cur_mm:
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08004000 ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
4001 if (ret == -EINTR)
4002 ret = 0;
4003 return ret;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004004}
4005
Johannes Weiner0435a2f2012-07-31 16:45:43 -07004006int mem_cgroup_try_charge_swapin(struct mm_struct *mm, struct page *page,
4007 gfp_t gfp_mask, struct mem_cgroup **memcgp)
4008{
4009 *memcgp = NULL;
4010 if (mem_cgroup_disabled())
4011 return 0;
Johannes Weinerbdf4f4d2012-07-31 16:45:50 -07004012 /*
4013 * A racing thread's fault, or swapoff, may have already
4014 * updated the pte, and even removed page from swap cache: in
4015 * those cases unuse_pte()'s pte_same() test will fail; but
4016 * there's also a KSM case which does need to charge the page.
4017 */
4018 if (!PageSwapCache(page)) {
4019 int ret;
4020
4021 ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, memcgp, true);
4022 if (ret == -EINTR)
4023 ret = 0;
4024 return ret;
4025 }
Johannes Weiner0435a2f2012-07-31 16:45:43 -07004026 return __mem_cgroup_try_charge_swapin(mm, page, gfp_mask, memcgp);
4027}
4028
Johannes Weiner827a03d2012-07-31 16:45:36 -07004029void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
4030{
4031 if (mem_cgroup_disabled())
4032 return;
4033 if (!memcg)
4034 return;
4035 __mem_cgroup_cancel_charge(memcg, 1);
4036}
4037
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07004038static void
Johannes Weiner72835c82012-01-12 17:18:32 -08004039__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07004040 enum charge_type ctype)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08004041{
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08004042 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08004043 return;
Johannes Weiner72835c82012-01-12 17:18:32 -08004044 if (!memcg)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08004045 return;
KAMEZAWA Hiroyuki5a6475a2011-03-23 16:42:42 -07004046
Johannes Weinerce587e62012-04-24 20:22:33 +02004047 __mem_cgroup_commit_charge(memcg, page, 1, ctype, true);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004048 /*
4049 * Now swap is on-memory. This means this page may be
4050 * counted both as mem and swap....double count.
KAMEZAWA Hiroyuki03f3c432009-01-07 18:08:31 -08004051 * Fix it by uncharging from memsw. Basically, this SwapCache is stable
4052 * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
4053 * may call delete_from_swap_cache() before reach here.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004054 */
KAMEZAWA Hiroyuki03f3c432009-01-07 18:08:31 -08004055 if (do_swap_account && PageSwapCache(page)) {
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004056 swp_entry_t ent = {.val = page_private(page)};
Hugh Dickins86493002012-05-29 15:06:52 -07004057 mem_cgroup_uncharge_swap(ent);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004058 }
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08004059}
4060
Johannes Weiner72835c82012-01-12 17:18:32 -08004061void mem_cgroup_commit_charge_swapin(struct page *page,
4062 struct mem_cgroup *memcg)
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07004063{
Johannes Weiner72835c82012-01-12 17:18:32 -08004064 __mem_cgroup_commit_charge_swapin(page, memcg,
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -07004065 MEM_CGROUP_CHARGE_TYPE_ANON);
Daisuke Nishimura83aae4c2009-04-02 16:57:48 -07004066}
4067
Johannes Weiner827a03d2012-07-31 16:45:36 -07004068int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
4069 gfp_t gfp_mask)
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08004070{
Johannes Weiner827a03d2012-07-31 16:45:36 -07004071 struct mem_cgroup *memcg = NULL;
4072 enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
4073 int ret;
4074
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08004075 if (mem_cgroup_disabled())
Johannes Weiner827a03d2012-07-31 16:45:36 -07004076 return 0;
4077 if (PageCompound(page))
4078 return 0;
4079
Johannes Weiner827a03d2012-07-31 16:45:36 -07004080 if (!PageSwapCache(page))
4081 ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
4082 else { /* page is swapcache/shmem */
Johannes Weiner0435a2f2012-07-31 16:45:43 -07004083 ret = __mem_cgroup_try_charge_swapin(mm, page,
4084 gfp_mask, &memcg);
Johannes Weiner827a03d2012-07-31 16:45:36 -07004085 if (!ret)
4086 __mem_cgroup_commit_charge_swapin(page, memcg, type);
4087 }
4088 return ret;
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08004089}
4090
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004091static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
Johannes Weiner7ec99d62011-03-23 16:42:36 -07004092 unsigned int nr_pages,
4093 const enum charge_type ctype)
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004094{
4095 struct memcg_batch_info *batch = NULL;
4096 bool uncharge_memsw = true;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07004097
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004098 /* If swapout, usage of swap doesn't decrease */
4099 if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
4100 uncharge_memsw = false;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004101
4102 batch = &current->memcg_batch;
4103 /*
4104 * In usual, we do css_get() when we remember memcg pointer.
4105 * But in this case, we keep res->usage until end of a series of
4106 * uncharges. Then, it's ok to ignore memcg's refcnt.
4107 */
4108 if (!batch->memcg)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004109 batch->memcg = memcg;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004110 /*
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004111 * do_batch > 0 when unmapping pages or inode invalidate/truncate.
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004112 * In those cases, all pages freed continuously can be expected to be in
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004113 * the same cgroup and we have chance to coalesce uncharges.
4114 * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
4115 * because we want to do uncharge as soon as possible.
4116 */
4117
4118 if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
4119 goto direct_uncharge;
4120
Johannes Weiner7ec99d62011-03-23 16:42:36 -07004121 if (nr_pages > 1)
Andrea Arcangeliec168512011-01-13 15:46:56 -08004122 goto direct_uncharge;
4123
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004124 /*
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004125 * In typical case, batch->memcg == mem. This means we can
4126 * merge a series of uncharges to an uncharge of res_counter.
4127 * If not, we uncharge res_counter ony by one.
4128 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004129 if (batch->memcg != memcg)
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004130 goto direct_uncharge;
4131 /* remember freed charge and uncharge it later */
Johannes Weiner7ffd4ca2011-03-23 16:42:35 -07004132 batch->nr_pages++;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004133 if (uncharge_memsw)
Johannes Weiner7ffd4ca2011-03-23 16:42:35 -07004134 batch->memsw_nr_pages++;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004135 return;
4136direct_uncharge:
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004137 res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004138 if (uncharge_memsw)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004139 res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
4140 if (unlikely(batch->memcg != memcg))
4141 memcg_oom_recover(memcg);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004142}
KAMEZAWA Hiroyuki7a81b882009-01-07 18:07:48 -08004143
Balbir Singh8697d332008-02-07 00:13:59 -08004144/*
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004145 * uncharge if !page_mapped(page)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08004146 */
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004147static struct mem_cgroup *
Johannes Weiner0030f532012-07-31 16:45:25 -07004148__mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype,
4149 bool end_migration)
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08004150{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004151 struct mem_cgroup *memcg = NULL;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07004152 unsigned int nr_pages = 1;
4153 struct page_cgroup *pc;
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07004154 bool anon;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08004155
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08004156 if (mem_cgroup_disabled())
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004157 return NULL;
Balbir Singh40779602008-04-04 14:29:59 -07004158
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08004159 if (PageTransHuge(page)) {
Johannes Weiner7ec99d62011-03-23 16:42:36 -07004160 nr_pages <<= compound_order(page);
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -08004161 VM_BUG_ON(!PageTransHuge(page));
4162 }
Balbir Singh8697d332008-02-07 00:13:59 -08004163 /*
Balbir Singh3c541e12008-02-07 00:14:41 -08004164 * Check if our page_cgroup is valid
Balbir Singh8697d332008-02-07 00:13:59 -08004165 */
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004166 pc = lookup_page_cgroup(page);
Johannes Weinercfa44942012-01-12 17:18:38 -08004167 if (unlikely(!PageCgroupUsed(pc)))
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004168 return NULL;
Balbir Singh8a9f3cc2008-02-07 00:13:53 -08004169
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004170 lock_page_cgroup(pc);
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004171
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004172 memcg = pc->mem_cgroup;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004173
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004174 if (!PageCgroupUsed(pc))
4175 goto unlock_out;
4176
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07004177 anon = PageAnon(page);
4178
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004179 switch (ctype) {
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -07004180 case MEM_CGROUP_CHARGE_TYPE_ANON:
KAMEZAWA Hiroyuki2ff76f12012-03-21 16:34:25 -07004181 /*
4182 * Generally PageAnon tells if it's the anon statistics to be
4183 * updated; but sometimes e.g. mem_cgroup_uncharge_page() is
4184 * used before page reached the stage of being marked PageAnon.
4185 */
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07004186 anon = true;
4187 /* fallthrough */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07004188 case MEM_CGROUP_CHARGE_TYPE_DROP:
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004189 /* See mem_cgroup_prepare_migration() */
Johannes Weiner0030f532012-07-31 16:45:25 -07004190 if (page_mapped(page))
4191 goto unlock_out;
4192 /*
4193 * Pages under migration may not be uncharged. But
4194 * end_migration() /must/ be the one uncharging the
4195 * unused post-migration page and so it has to call
4196 * here with the migration bit still set. See the
4197 * res_counter handling below.
4198 */
4199 if (!end_migration && PageCgroupMigration(pc))
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004200 goto unlock_out;
4201 break;
4202 case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
4203 if (!PageAnon(page)) { /* Shared memory */
4204 if (page->mapping && !page_is_file_cache(page))
4205 goto unlock_out;
4206 } else if (page_mapped(page)) /* Anon */
4207 goto unlock_out;
4208 break;
4209 default:
4210 break;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004211 }
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004212
David Rientjesb070e652013-05-07 16:18:09 -07004213 mem_cgroup_charge_statistics(memcg, page, anon, -nr_pages);
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07004214
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004215 ClearPageCgroupUsed(pc);
KAMEZAWA Hiroyuki544122e2009-01-07 18:08:34 -08004216 /*
4217 * pc->mem_cgroup is not cleared here. It will be accessed when it's
4218 * freed from LRU. This is safe because uncharged page is expected not
4219 * to be reused (freed soon). Exception is SwapCache, it's handled by
4220 * special functions.
4221 */
Hugh Dickinsb9c565d2008-03-04 14:29:11 -08004222
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004223 unlock_page_cgroup(pc);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07004224 /*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004225 * even after unlock, we have memcg->res.usage here and this memcg
Li Zefan40503772013-07-08 16:00:34 -07004226 * will never be freed, so it's safe to call css_get().
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07004227 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004228 memcg_check_events(memcg, page);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07004229 if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004230 mem_cgroup_swap_statistics(memcg, true);
Li Zefan40503772013-07-08 16:00:34 -07004231 css_get(&memcg->css);
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07004232 }
Johannes Weiner0030f532012-07-31 16:45:25 -07004233 /*
4234 * Migration does not charge the res_counter for the
4235 * replacement page, so leave it alone when phasing out the
4236 * page that is unused after the migration.
4237 */
4238 if (!end_migration && !mem_cgroup_is_root(memcg))
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004239 mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08004240
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004241 return memcg;
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004242
4243unlock_out:
4244 unlock_page_cgroup(pc);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004245 return NULL;
Balbir Singh3c541e12008-02-07 00:14:41 -08004246}
4247
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004248void mem_cgroup_uncharge_page(struct page *page)
4249{
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004250 /* early check. */
4251 if (page_mapped(page))
4252 return;
Johannes Weiner40f23a22012-01-12 17:18:45 -08004253 VM_BUG_ON(page->mapping && !PageAnon(page));
Johannes Weiner28ccddf2013-05-24 15:55:15 -07004254 /*
4255 * If the page is in swap cache, uncharge should be deferred
4256 * to the swap path, which also properly accounts swap usage
4257 * and handles memcg lifetime.
4258 *
4259 * Note that this check is not stable and reclaim may add the
4260 * page to swap cache at any time after this. However, if the
4261 * page is not in swap cache by the time page->mapcount hits
4262 * 0, there won't be any page table references to the swap
4263 * slot, and reclaim will free it and not actually write the
4264 * page to disk.
4265 */
Johannes Weiner0c59b892012-07-31 16:45:31 -07004266 if (PageSwapCache(page))
4267 return;
Johannes Weiner0030f532012-07-31 16:45:25 -07004268 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_ANON, false);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004269}
4270
4271void mem_cgroup_uncharge_cache_page(struct page *page)
4272{
4273 VM_BUG_ON(page_mapped(page));
KAMEZAWA Hiroyukib7abea92008-10-18 20:28:09 -07004274 VM_BUG_ON(page->mapping);
Johannes Weiner0030f532012-07-31 16:45:25 -07004275 __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE, false);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004276}
4277
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004278/*
4279 * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
4280 * In that cases, pages are freed continuously and we can expect pages
4281 * are in the same memcg. All these calls itself limits the number of
4282 * pages freed at once, then uncharge_start/end() is called properly.
4283 * This may be called prural(2) times in a context,
4284 */
4285
4286void mem_cgroup_uncharge_start(void)
4287{
4288 current->memcg_batch.do_batch++;
4289 /* We can do nest. */
4290 if (current->memcg_batch.do_batch == 1) {
4291 current->memcg_batch.memcg = NULL;
Johannes Weiner7ffd4ca2011-03-23 16:42:35 -07004292 current->memcg_batch.nr_pages = 0;
4293 current->memcg_batch.memsw_nr_pages = 0;
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004294 }
4295}
4296
4297void mem_cgroup_uncharge_end(void)
4298{
4299 struct memcg_batch_info *batch = &current->memcg_batch;
4300
4301 if (!batch->do_batch)
4302 return;
4303
4304 batch->do_batch--;
4305 if (batch->do_batch) /* If stacked, do nothing. */
4306 return;
4307
4308 if (!batch->memcg)
4309 return;
4310 /*
4311 * This "batch->memcg" is valid without any css_get/put etc...
4312 * bacause we hide charges behind us.
4313 */
Johannes Weiner7ffd4ca2011-03-23 16:42:35 -07004314 if (batch->nr_pages)
4315 res_counter_uncharge(&batch->memcg->res,
4316 batch->nr_pages * PAGE_SIZE);
4317 if (batch->memsw_nr_pages)
4318 res_counter_uncharge(&batch->memcg->memsw,
4319 batch->memsw_nr_pages * PAGE_SIZE);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004320 memcg_oom_recover(batch->memcg);
KAMEZAWA Hiroyuki569b8462009-12-15 16:47:03 -08004321 /* forget this pointer (for sanity check) */
4322 batch->memcg = NULL;
4323}
4324
Daisuke Nishimurae767e052009-05-28 14:34:28 -07004325#ifdef CONFIG_SWAP
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004326/*
Daisuke Nishimurae767e052009-05-28 14:34:28 -07004327 * called after __delete_from_swap_cache() and drop "page" account.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004328 * memcg information is recorded to swap_cgroup of "ent"
4329 */
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07004330void
4331mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004332{
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004333 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07004334 int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004335
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07004336 if (!swapout) /* this was a swap cache but the swap is unused ! */
4337 ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
4338
Johannes Weiner0030f532012-07-31 16:45:25 -07004339 memcg = __mem_cgroup_uncharge_common(page, ctype, false);
KAMEZAWA Hiroyuki8a9478c2009-06-17 16:27:17 -07004340
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07004341 /*
4342 * record memcg information, if swapout && memcg != NULL,
Li Zefan40503772013-07-08 16:00:34 -07004343 * css_get() was called in uncharge().
KAMEZAWA Hiroyukif75ca962010-08-10 18:03:02 -07004344 */
4345 if (do_swap_account && swapout && memcg)
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07004346 swap_cgroup_record(ent, css_id(&memcg->css));
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004347}
Daisuke Nishimurae767e052009-05-28 14:34:28 -07004348#endif
KAMEZAWA Hiroyukid13d1442009-01-07 18:07:56 -08004349
Andrew Mortonc255a452012-07-31 16:43:02 -07004350#ifdef CONFIG_MEMCG_SWAP
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004351/*
4352 * called from swap_entry_free(). remove record in swap_cgroup and
4353 * uncharge "memsw" account.
4354 */
4355void mem_cgroup_uncharge_swap(swp_entry_t ent)
4356{
4357 struct mem_cgroup *memcg;
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07004358 unsigned short id;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004359
4360 if (!do_swap_account)
4361 return;
4362
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07004363 id = swap_cgroup_record(ent, 0);
4364 rcu_read_lock();
4365 memcg = mem_cgroup_lookup(id);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004366 if (memcg) {
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07004367 /*
4368 * We uncharge this because swap is freed.
4369 * This memcg can be obsolete one. We avoid calling css_tryget
4370 */
Balbir Singh0c3e73e2009-09-23 15:56:42 -07004371 if (!mem_cgroup_is_root(memcg))
KAMEZAWA Hiroyuki4e649152009-10-01 15:44:11 -07004372 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
Balbir Singh0c3e73e2009-09-23 15:56:42 -07004373 mem_cgroup_swap_statistics(memcg, false);
Li Zefan40503772013-07-08 16:00:34 -07004374 css_put(&memcg->css);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004375 }
KAMEZAWA Hiroyukia3b2d692009-04-02 16:57:45 -07004376 rcu_read_unlock();
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004377}
Daisuke Nishimura02491442010-03-10 15:22:17 -08004378
4379/**
4380 * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
4381 * @entry: swap entry to be moved
4382 * @from: mem_cgroup which the entry is moved from
4383 * @to: mem_cgroup which the entry is moved to
4384 *
4385 * It succeeds only when the swap_cgroup's record for this entry is the same
4386 * as the mem_cgroup's id of @from.
4387 *
4388 * Returns 0 on success, -EINVAL on failure.
4389 *
4390 * The caller must have charged to @to, IOW, called res_counter_charge() about
4391 * both res and memsw, and called css_get().
4392 */
4393static int mem_cgroup_move_swap_account(swp_entry_t entry,
Hugh Dickinse91cbb42012-05-29 15:06:51 -07004394 struct mem_cgroup *from, struct mem_cgroup *to)
Daisuke Nishimura02491442010-03-10 15:22:17 -08004395{
4396 unsigned short old_id, new_id;
4397
4398 old_id = css_id(&from->css);
4399 new_id = css_id(&to->css);
4400
4401 if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08004402 mem_cgroup_swap_statistics(from, false);
Daisuke Nishimura02491442010-03-10 15:22:17 -08004403 mem_cgroup_swap_statistics(to, true);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004404 /*
4405 * This function is only called from task migration context now.
4406 * It postpones res_counter and refcount handling till the end
4407 * of task migration(mem_cgroup_clear_mc()) for performance
Li Zefan40503772013-07-08 16:00:34 -07004408 * improvement. But we cannot postpone css_get(to) because if
4409 * the process that has been moved to @to does swap-in, the
4410 * refcount of @to might be decreased to 0.
4411 *
4412 * We are in attach() phase, so the cgroup is guaranteed to be
4413 * alive, so we can just call css_get().
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08004414 */
Li Zefan40503772013-07-08 16:00:34 -07004415 css_get(&to->css);
Daisuke Nishimura02491442010-03-10 15:22:17 -08004416 return 0;
4417 }
4418 return -EINVAL;
4419}
4420#else
4421static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
Hugh Dickinse91cbb42012-05-29 15:06:51 -07004422 struct mem_cgroup *from, struct mem_cgroup *to)
Daisuke Nishimura02491442010-03-10 15:22:17 -08004423{
4424 return -EINVAL;
4425}
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004426#endif
4427
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08004428/*
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004429 * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
4430 * page belongs to.
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08004431 */
Johannes Weiner0030f532012-07-31 16:45:25 -07004432void mem_cgroup_prepare_migration(struct page *page, struct page *newpage,
4433 struct mem_cgroup **memcgp)
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08004434{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004435 struct mem_cgroup *memcg = NULL;
Mel Gormanb32967f2012-11-19 12:35:47 +00004436 unsigned int nr_pages = 1;
Johannes Weiner7ec99d62011-03-23 16:42:36 -07004437 struct page_cgroup *pc;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004438 enum charge_type ctype;
Hugh Dickins8869b8f2008-03-04 14:29:09 -08004439
Johannes Weiner72835c82012-01-12 17:18:32 -08004440 *memcgp = NULL;
KAMEZAWA Hiroyuki56039ef2011-03-23 16:42:19 -07004441
Hirokazu Takahashif8d665422009-01-07 18:08:02 -08004442 if (mem_cgroup_disabled())
Johannes Weiner0030f532012-07-31 16:45:25 -07004443 return;
Balbir Singh40779602008-04-04 14:29:59 -07004444
Mel Gormanb32967f2012-11-19 12:35:47 +00004445 if (PageTransHuge(page))
4446 nr_pages <<= compound_order(page);
4447
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004448 pc = lookup_page_cgroup(page);
4449 lock_page_cgroup(pc);
4450 if (PageCgroupUsed(pc)) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004451 memcg = pc->mem_cgroup;
4452 css_get(&memcg->css);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004453 /*
4454 * At migrating an anonymous page, its mapcount goes down
4455 * to 0 and uncharge() will be called. But, even if it's fully
4456 * unmapped, migration may fail and this page has to be
4457 * charged again. We set MIGRATION flag here and delay uncharge
4458 * until end_migration() is called
4459 *
4460 * Corner Case Thinking
4461 * A)
4462 * When the old page was mapped as Anon and it's unmap-and-freed
4463 * while migration was ongoing.
4464 * If unmap finds the old page, uncharge() of it will be delayed
4465 * until end_migration(). If unmap finds a new page, it's
4466 * uncharged when it make mapcount to be 1->0. If unmap code
4467 * finds swap_migration_entry, the new page will not be mapped
4468 * and end_migration() will find it(mapcount==0).
4469 *
4470 * B)
4471 * When the old page was mapped but migraion fails, the kernel
4472 * remaps it. A charge for it is kept by MIGRATION flag even
4473 * if mapcount goes down to 0. We can do remap successfully
4474 * without charging it again.
4475 *
4476 * C)
4477 * The "old" page is under lock_page() until the end of
4478 * migration, so, the old page itself will not be swapped-out.
4479 * If the new page is swapped out before end_migraton, our
4480 * hook to usual swap-out path will catch the event.
4481 */
4482 if (PageAnon(page))
4483 SetPageCgroupMigration(pc);
Hugh Dickinsb9c565d2008-03-04 14:29:11 -08004484 }
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004485 unlock_page_cgroup(pc);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004486 /*
4487 * If the page is not charged at this point,
4488 * we return here.
4489 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004490 if (!memcg)
Johannes Weiner0030f532012-07-31 16:45:25 -07004491 return;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004492
Johannes Weiner72835c82012-01-12 17:18:32 -08004493 *memcgp = memcg;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004494 /*
4495 * We charge new page before it's used/mapped. So, even if unlock_page()
4496 * is called before end_migration, we can catch all events on this new
4497 * page. In the case new page is migrated but not remapped, new page's
4498 * mapcount will be finally 0 and we call uncharge in end_migration().
4499 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004500 if (PageAnon(page))
Kamezawa Hiroyuki41326c12012-07-31 16:41:40 -07004501 ctype = MEM_CGROUP_CHARGE_TYPE_ANON;
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004502 else
Johannes Weiner62ba7442012-07-31 16:45:39 -07004503 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
Johannes Weiner0030f532012-07-31 16:45:25 -07004504 /*
4505 * The page is committed to the memcg, but it's not actually
4506 * charged to the res_counter since we plan on replacing the
4507 * old one and only one page is going to be left afterwards.
4508 */
Mel Gormanb32967f2012-11-19 12:35:47 +00004509 __mem_cgroup_commit_charge(memcg, newpage, nr_pages, ctype, false);
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07004510}
Hugh Dickinsfb59e9f2008-03-04 14:29:16 -08004511
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004512/* remove redundant charge if migration failed*/
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004513void mem_cgroup_end_migration(struct mem_cgroup *memcg,
Daisuke Nishimura50de1dd2011-01-13 15:47:43 -08004514 struct page *oldpage, struct page *newpage, bool migration_ok)
KAMEZAWA Hiroyukie8589cc2008-07-25 01:47:10 -07004515{
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004516 struct page *used, *unused;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004517 struct page_cgroup *pc;
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07004518 bool anon;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004519
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004520 if (!memcg)
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004521 return;
Tejun Heob25ed602012-11-05 09:16:59 -08004522
Daisuke Nishimura50de1dd2011-01-13 15:47:43 -08004523 if (!migration_ok) {
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004524 used = oldpage;
4525 unused = newpage;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004526 } else {
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004527 used = newpage;
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004528 unused = oldpage;
4529 }
Johannes Weiner0030f532012-07-31 16:45:25 -07004530 anon = PageAnon(used);
Johannes Weiner7d188952012-07-31 16:45:34 -07004531 __mem_cgroup_uncharge_common(unused,
4532 anon ? MEM_CGROUP_CHARGE_TYPE_ANON
4533 : MEM_CGROUP_CHARGE_TYPE_CACHE,
4534 true);
Johannes Weiner0030f532012-07-31 16:45:25 -07004535 css_put(&memcg->css);
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004536 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004537 * We disallowed uncharge of pages under migration because mapcount
4538 * of the page goes down to zero, temporarly.
4539 * Clear the flag and check the page should be charged.
KAMEZAWA Hiroyuki69029cd2008-07-25 01:47:14 -07004540 */
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004541 pc = lookup_page_cgroup(oldpage);
4542 lock_page_cgroup(pc);
4543 ClearPageCgroupMigration(pc);
4544 unlock_page_cgroup(pc);
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004545
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004546 /*
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004547 * If a page is a file cache, radix-tree replacement is very atomic
4548 * and we can skip this check. When it was an Anon page, its mapcount
4549 * goes down to 0. But because we added MIGRATION flage, it's not
4550 * uncharged yet. There are several case but page->mapcount check
4551 * and USED bit check in mem_cgroup_uncharge_page() will do enough
4552 * check. (see prepare_charge() also)
KAMEZAWA Hiroyuki01b1ae62009-01-07 18:07:50 -08004553 */
KAMEZAWA Hiroyukib24028572012-03-21 16:34:22 -07004554 if (anon)
akpm@linux-foundation.orgac39cf82010-05-26 14:42:46 -07004555 mem_cgroup_uncharge_page(used);
KAMEZAWA Hiroyukiae41be32008-02-07 00:14:10 -08004556}
Pavel Emelianov78fb7462008-02-07 00:13:51 -08004557
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004558/*
4559 * At replace page cache, newpage is not under any memcg but it's on
4560 * LRU. So, this function doesn't touch res_counter but handles LRU
4561 * in correct way. Both pages are locked so we cannot race with uncharge.
4562 */
4563void mem_cgroup_replace_page_cache(struct page *oldpage,
4564 struct page *newpage)
4565{
Hugh Dickinsbde05d12012-05-29 15:06:38 -07004566 struct mem_cgroup *memcg = NULL;
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004567 struct page_cgroup *pc;
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004568 enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004569
4570 if (mem_cgroup_disabled())
4571 return;
4572
4573 pc = lookup_page_cgroup(oldpage);
4574 /* fix accounting on old pages */
4575 lock_page_cgroup(pc);
Hugh Dickinsbde05d12012-05-29 15:06:38 -07004576 if (PageCgroupUsed(pc)) {
4577 memcg = pc->mem_cgroup;
David Rientjesb070e652013-05-07 16:18:09 -07004578 mem_cgroup_charge_statistics(memcg, oldpage, false, -1);
Hugh Dickinsbde05d12012-05-29 15:06:38 -07004579 ClearPageCgroupUsed(pc);
4580 }
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004581 unlock_page_cgroup(pc);
4582
Hugh Dickinsbde05d12012-05-29 15:06:38 -07004583 /*
4584 * When called from shmem_replace_page(), in some cases the
4585 * oldpage has already been charged, and in some cases not.
4586 */
4587 if (!memcg)
4588 return;
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004589 /*
4590 * Even if newpage->mapping was NULL before starting replacement,
4591 * the newpage may be on LRU(or pagevec for LRU) already. We lock
4592 * LRU while we overwrite pc->mem_cgroup.
4593 */
Johannes Weinerce587e62012-04-24 20:22:33 +02004594 __mem_cgroup_commit_charge(memcg, newpage, 1, type, true);
KAMEZAWA Hiroyukiab936cb2012-01-12 17:17:44 -08004595}
4596
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -07004597#ifdef CONFIG_DEBUG_VM
4598static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
4599{
4600 struct page_cgroup *pc;
4601
4602 pc = lookup_page_cgroup(page);
Johannes Weinercfa44942012-01-12 17:18:38 -08004603 /*
4604 * Can be NULL while feeding pages into the page allocator for
4605 * the first time, i.e. during boot or memory hotplug;
4606 * or when mem_cgroup_disabled().
4607 */
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -07004608 if (likely(pc) && PageCgroupUsed(pc))
4609 return pc;
4610 return NULL;
4611}
4612
4613bool mem_cgroup_bad_page_check(struct page *page)
4614{
4615 if (mem_cgroup_disabled())
4616 return false;
4617
4618 return lookup_page_cgroup_used(page) != NULL;
4619}
4620
4621void mem_cgroup_print_bad_page(struct page *page)
4622{
4623 struct page_cgroup *pc;
4624
4625 pc = lookup_page_cgroup_used(page);
4626 if (pc) {
Andrew Mortond0451972013-02-22 16:32:06 -08004627 pr_alert("pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
4628 pc, pc->flags, pc->mem_cgroup);
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -07004629 }
4630}
4631#endif
4632
KOSAKI Motohirod38d2a72009-01-06 14:39:44 -08004633static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004634 unsigned long long val)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004635{
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004636 int retry_count;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004637 u64 memswlimit, memlimit;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004638 int ret = 0;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004639 int children = mem_cgroup_count_children(memcg);
4640 u64 curusage, oldusage;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004641 int enlarge;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004642
4643 /*
4644 * For keeping hierarchical_reclaim simple, how long we should retry
4645 * is depends on callers. We set our retry-count to be function
4646 * of # of children which we should visit in this loop.
4647 */
4648 retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
4649
4650 oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004651
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004652 enlarge = 0;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004653 while (retry_count) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004654 if (signal_pending(current)) {
4655 ret = -EINTR;
4656 break;
4657 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004658 /*
4659 * Rather than hide all in some function, I do this in
4660 * open coded manner. You see what this really does.
Wanpeng Liaaad1532012-07-31 16:43:23 -07004661 * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004662 */
4663 mutex_lock(&set_limit_mutex);
4664 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4665 if (memswlimit < val) {
4666 ret = -EINVAL;
4667 mutex_unlock(&set_limit_mutex);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004668 break;
4669 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004670
4671 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4672 if (memlimit < val)
4673 enlarge = 1;
4674
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004675 ret = res_counter_set_limit(&memcg->res, val);
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07004676 if (!ret) {
4677 if (memswlimit == val)
4678 memcg->memsw_is_minimum = true;
4679 else
4680 memcg->memsw_is_minimum = false;
4681 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004682 mutex_unlock(&set_limit_mutex);
4683
4684 if (!ret)
4685 break;
4686
Johannes Weiner56600482012-01-12 17:17:59 -08004687 mem_cgroup_reclaim(memcg, GFP_KERNEL,
4688 MEM_CGROUP_RECLAIM_SHRINK);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004689 curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
4690 /* Usage is reduced ? */
Andrew Mortonf894ffa2013-09-12 15:13:35 -07004691 if (curusage >= oldusage)
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004692 retry_count--;
4693 else
4694 oldusage = curusage;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004695 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004696 if (!ret && enlarge)
4697 memcg_oom_recover(memcg);
KOSAKI Motohiro14797e22009-01-07 18:08:18 -08004698
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004699 return ret;
4700}
4701
Li Zefan338c8432009-06-17 16:27:15 -07004702static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
4703 unsigned long long val)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004704{
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004705 int retry_count;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004706 u64 memlimit, memswlimit, oldusage, curusage;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004707 int children = mem_cgroup_count_children(memcg);
4708 int ret = -EBUSY;
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004709 int enlarge = 0;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004710
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004711 /* see mem_cgroup_resize_res_limit */
Andrew Mortonf894ffa2013-09-12 15:13:35 -07004712 retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004713 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004714 while (retry_count) {
4715 if (signal_pending(current)) {
4716 ret = -EINTR;
4717 break;
4718 }
4719 /*
4720 * Rather than hide all in some function, I do this in
4721 * open coded manner. You see what this really does.
Wanpeng Liaaad1532012-07-31 16:43:23 -07004722 * We have to guarantee memcg->res.limit <= memcg->memsw.limit.
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004723 */
4724 mutex_lock(&set_limit_mutex);
4725 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
4726 if (memlimit > val) {
4727 ret = -EINVAL;
4728 mutex_unlock(&set_limit_mutex);
4729 break;
4730 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004731 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
4732 if (memswlimit < val)
4733 enlarge = 1;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004734 ret = res_counter_set_limit(&memcg->memsw, val);
KAMEZAWA Hiroyuki22a668d2009-06-17 16:27:19 -07004735 if (!ret) {
4736 if (memlimit == val)
4737 memcg->memsw_is_minimum = true;
4738 else
4739 memcg->memsw_is_minimum = false;
4740 }
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004741 mutex_unlock(&set_limit_mutex);
4742
4743 if (!ret)
4744 break;
4745
Johannes Weiner56600482012-01-12 17:17:59 -08004746 mem_cgroup_reclaim(memcg, GFP_KERNEL,
4747 MEM_CGROUP_RECLAIM_NOSWAP |
4748 MEM_CGROUP_RECLAIM_SHRINK);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004749 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004750 /* Usage is reduced ? */
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08004751 if (curusage >= oldusage)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004752 retry_count--;
KAMEZAWA Hiroyuki81d39c22009-04-02 16:57:36 -07004753 else
4754 oldusage = curusage;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004755 }
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07004756 if (!ret && enlarge)
4757 memcg_oom_recover(memcg);
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07004758 return ret;
4759}
4760
Andrew Morton0608f432013-09-24 15:27:41 -07004761unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
4762 gfp_t gfp_mask,
4763 unsigned long *total_scanned)
4764{
4765 unsigned long nr_reclaimed = 0;
4766 struct mem_cgroup_per_zone *mz, *next_mz = NULL;
4767 unsigned long reclaimed;
4768 int loop = 0;
4769 struct mem_cgroup_tree_per_zone *mctz;
4770 unsigned long long excess;
4771 unsigned long nr_scanned;
4772
4773 if (order > 0)
4774 return 0;
4775
4776 mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
4777 /*
4778 * This loop can run a while, specially if mem_cgroup's continuously
4779 * keep exceeding their soft limit and putting the system under
4780 * pressure
4781 */
4782 do {
4783 if (next_mz)
4784 mz = next_mz;
4785 else
4786 mz = mem_cgroup_largest_soft_limit_node(mctz);
4787 if (!mz)
4788 break;
4789
4790 nr_scanned = 0;
4791 reclaimed = mem_cgroup_soft_reclaim(mz->memcg, zone,
4792 gfp_mask, &nr_scanned);
4793 nr_reclaimed += reclaimed;
4794 *total_scanned += nr_scanned;
4795 spin_lock(&mctz->lock);
4796
4797 /*
4798 * If we failed to reclaim anything from this memory cgroup
4799 * it is time to move on to the next cgroup
4800 */
4801 next_mz = NULL;
4802 if (!reclaimed) {
4803 do {
4804 /*
4805 * Loop until we find yet another one.
4806 *
4807 * By the time we get the soft_limit lock
4808 * again, someone might have aded the
4809 * group back on the RB tree. Iterate to
4810 * make sure we get a different mem.
4811 * mem_cgroup_largest_soft_limit_node returns
4812 * NULL if no other cgroup is present on
4813 * the tree
4814 */
4815 next_mz =
4816 __mem_cgroup_largest_soft_limit_node(mctz);
4817 if (next_mz == mz)
4818 css_put(&next_mz->memcg->css);
4819 else /* next_mz == NULL or other memcg */
4820 break;
4821 } while (1);
4822 }
4823 __mem_cgroup_remove_exceeded(mz->memcg, mz, mctz);
4824 excess = res_counter_soft_limit_excess(&mz->memcg->res);
4825 /*
4826 * One school of thought says that we should not add
4827 * back the node to the tree if reclaim returns 0.
4828 * But our reclaim could return 0, simply because due
4829 * to priority we are exposing a smaller subset of
4830 * memory to reclaim from. Consider this as a longer
4831 * term TODO.
4832 */
4833 /* If excess == 0, no tree ops */
4834 __mem_cgroup_insert_exceeded(mz->memcg, mz, mctz, excess);
4835 spin_unlock(&mctz->lock);
4836 css_put(&mz->memcg->css);
4837 loop++;
4838 /*
4839 * Could not reclaim anything and there are no more
4840 * mem cgroups to try or we seem to be looping without
4841 * reclaiming anything.
4842 */
4843 if (!nr_reclaimed &&
4844 (next_mz == NULL ||
4845 loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
4846 break;
4847 } while (!nr_reclaimed);
4848 if (next_mz)
4849 css_put(&next_mz->memcg->css);
4850 return nr_reclaimed;
4851}
4852
Michal Hocko2ef37d32012-10-26 13:37:30 +02004853/**
4854 * mem_cgroup_force_empty_list - clears LRU of a group
4855 * @memcg: group to clear
4856 * @node: NUMA node
4857 * @zid: zone id
4858 * @lru: lru to to clear
4859 *
KAMEZAWA Hiroyuki3c935d12012-07-31 16:42:46 -07004860 * Traverse a specified page_cgroup list and try to drop them all. This doesn't
Michal Hocko2ef37d32012-10-26 13:37:30 +02004861 * reclaim the pages page themselves - pages are moved to the parent (or root)
4862 * group.
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004863 */
Michal Hocko2ef37d32012-10-26 13:37:30 +02004864static void mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004865 int node, int zid, enum lru_list lru)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004866{
Hugh Dickinsbea8c152012-11-16 14:14:54 -08004867 struct lruvec *lruvec;
Michal Hocko2ef37d32012-10-26 13:37:30 +02004868 unsigned long flags;
KAMEZAWA Hiroyuki072c56c12008-02-07 00:14:39 -08004869 struct list_head *list;
Johannes Weiner925b7672012-01-12 17:18:15 -08004870 struct page *busy;
4871 struct zone *zone;
KAMEZAWA Hiroyuki072c56c12008-02-07 00:14:39 -08004872
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004873 zone = &NODE_DATA(node)->node_zones[zid];
Hugh Dickinsbea8c152012-11-16 14:14:54 -08004874 lruvec = mem_cgroup_zone_lruvec(zone, memcg);
4875 list = &lruvec->lists[lru];
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004876
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004877 busy = NULL;
Michal Hocko2ef37d32012-10-26 13:37:30 +02004878 do {
Johannes Weiner925b7672012-01-12 17:18:15 -08004879 struct page_cgroup *pc;
Johannes Weiner5564e882011-03-23 16:42:29 -07004880 struct page *page;
4881
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004882 spin_lock_irqsave(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004883 if (list_empty(list)) {
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004884 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004885 break;
4886 }
Johannes Weiner925b7672012-01-12 17:18:15 -08004887 page = list_entry(list->prev, struct page, lru);
4888 if (busy == page) {
4889 list_move(&page->lru, list);
Thiago Farina648bcc72010-03-05 13:42:04 -08004890 busy = NULL;
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004891 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004892 continue;
4893 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08004894 spin_unlock_irqrestore(&zone->lru_lock, flags);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004895
Johannes Weiner925b7672012-01-12 17:18:15 -08004896 pc = lookup_page_cgroup(page);
Johannes Weiner5564e882011-03-23 16:42:29 -07004897
KAMEZAWA Hiroyuki3c935d12012-07-31 16:42:46 -07004898 if (mem_cgroup_move_parent(page, pc, memcg)) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004899 /* found lock contention or "pc" is obsolete. */
Johannes Weiner925b7672012-01-12 17:18:15 -08004900 busy = page;
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004901 cond_resched();
4902 } else
4903 busy = NULL;
Michal Hocko2ef37d32012-10-26 13:37:30 +02004904 } while (!list_empty(list));
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004905}
4906
4907/*
Michal Hockoc26251f2012-10-26 13:37:28 +02004908 * make mem_cgroup's charge to be 0 if there is no task by moving
4909 * all the charges and pages to the parent.
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004910 * This enables deleting this mem_cgroup.
Michal Hockoc26251f2012-10-26 13:37:28 +02004911 *
4912 * Caller is responsible for holding css reference on the memcg.
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004913 */
Michal Hockoab5196c2012-10-26 13:37:32 +02004914static void mem_cgroup_reparent_charges(struct mem_cgroup *memcg)
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08004915{
Michal Hockoc26251f2012-10-26 13:37:28 +02004916 int node, zid;
Glauber Costabea207c2012-12-18 14:22:11 -08004917 u64 usage;
Hugh Dickins8869b8f2008-03-04 14:29:09 -08004918
Daisuke Nishimurafce66472010-01-15 17:01:30 -08004919 do {
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004920 /* This is for making all *used* pages to be on LRU. */
4921 lru_add_drain_all();
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004922 drain_all_stock_sync(memcg);
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004923 mem_cgroup_start_move(memcg);
Lai Jiangshan31aaea42012-12-12 13:51:27 -08004924 for_each_node_state(node, N_MEMORY) {
Michal Hocko2ef37d32012-10-26 13:37:30 +02004925 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
Hugh Dickinsf156ab92012-03-21 16:34:19 -07004926 enum lru_list lru;
4927 for_each_lru(lru) {
Michal Hocko2ef37d32012-10-26 13:37:30 +02004928 mem_cgroup_force_empty_list(memcg,
Hugh Dickinsf156ab92012-03-21 16:34:19 -07004929 node, zid, lru);
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004930 }
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08004931 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004932 }
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004933 mem_cgroup_end_move(memcg);
4934 memcg_oom_recover(memcg);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004935 cond_resched();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004936
Michal Hocko2ef37d32012-10-26 13:37:30 +02004937 /*
Glauber Costabea207c2012-12-18 14:22:11 -08004938 * Kernel memory may not necessarily be trackable to a specific
4939 * process. So they are not migrated, and therefore we can't
4940 * expect their value to drop to 0 here.
4941 * Having res filled up with kmem only is enough.
4942 *
Michal Hocko2ef37d32012-10-26 13:37:30 +02004943 * This is a safety check because mem_cgroup_force_empty_list
4944 * could have raced with mem_cgroup_replace_page_cache callers
4945 * so the lru seemed empty but the page could have been added
4946 * right after the check. RES_USAGE should be safe as we always
4947 * charge before adding to the LRU.
4948 */
Glauber Costabea207c2012-12-18 14:22:11 -08004949 usage = res_counter_read_u64(&memcg->res, RES_USAGE) -
4950 res_counter_read_u64(&memcg->kmem, RES_USAGE);
4951 } while (usage > 0);
Michal Hockoc26251f2012-10-26 13:37:28 +02004952}
4953
Glauber Costab5f99b52013-02-22 16:34:53 -08004954static inline bool memcg_has_children(struct mem_cgroup *memcg)
4955{
Johannes Weiner696ac172013-10-31 16:34:15 -07004956 lockdep_assert_held(&memcg_create_mutex);
4957 /*
4958 * The lock does not prevent addition or deletion to the list
4959 * of children, but it prevents a new child from being
4960 * initialized based on this parent in css_online(), so it's
4961 * enough to decide whether hierarchically inherited
4962 * attributes can still be changed or not.
4963 */
4964 return memcg->use_hierarchy &&
4965 !list_empty(&memcg->css.cgroup->children);
Glauber Costab5f99b52013-02-22 16:34:53 -08004966}
4967
4968/*
Michal Hockoc26251f2012-10-26 13:37:28 +02004969 * Reclaims as many pages from the given memcg as possible and moves
4970 * the rest to the parent.
4971 *
4972 * Caller is responsible for holding css reference for memcg.
4973 */
4974static int mem_cgroup_force_empty(struct mem_cgroup *memcg)
4975{
4976 int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
4977 struct cgroup *cgrp = memcg->css.cgroup;
4978
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004979 /* returns EBUSY if there is a task or if we come here twice. */
Michal Hockoc26251f2012-10-26 13:37:28 +02004980 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
4981 return -EBUSY;
4982
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004983 /* we call try-to-free pages for make this cgroup empty */
4984 lru_add_drain_all();
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004985 /* try to free all pages in this cgroup */
Glauber Costa569530f2012-04-12 12:49:13 -07004986 while (nr_retries && res_counter_read_u64(&memcg->res, RES_USAGE) > 0) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004987 int progress;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004988
Michal Hockoc26251f2012-10-26 13:37:28 +02004989 if (signal_pending(current))
4990 return -EINTR;
4991
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07004992 progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
Johannes Weiner185efc02011-09-14 16:21:58 -07004993 false);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004994 if (!progress) {
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004995 nr_retries--;
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004996 /* maybe some writeback is necessary */
Jens Axboe8aa7e842009-07-09 14:52:32 +02004997 congestion_wait(BLK_RW_ASYNC, HZ/10);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08004998 }
KAMEZAWA Hiroyukif817ed42009-01-07 18:07:53 -08004999
5000 }
KAMEZAWA Hiroyuki08e552c2009-01-07 18:08:01 -08005001 lru_add_drain();
Michal Hockoab5196c2012-10-26 13:37:32 +02005002 mem_cgroup_reparent_charges(memcg);
5003
5004 return 0;
KAMEZAWA Hiroyukicc847582008-02-07 00:14:16 -08005005}
5006
Tejun Heo182446d2013-08-08 20:11:24 -04005007static int mem_cgroup_force_empty_write(struct cgroup_subsys_state *css,
5008 unsigned int event)
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08005009{
Tejun Heo182446d2013-08-08 20:11:24 -04005010 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Michal Hockoc26251f2012-10-26 13:37:28 +02005011
Michal Hockod8423012012-10-26 13:37:29 +02005012 if (mem_cgroup_is_root(memcg))
5013 return -EINVAL;
Li Zefanc33bd832013-09-12 15:13:19 -07005014 return mem_cgroup_force_empty(memcg);
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08005015}
5016
Tejun Heo182446d2013-08-08 20:11:24 -04005017static u64 mem_cgroup_hierarchy_read(struct cgroup_subsys_state *css,
5018 struct cftype *cft)
Balbir Singh18f59ea2009-01-07 18:08:07 -08005019{
Tejun Heo182446d2013-08-08 20:11:24 -04005020 return mem_cgroup_from_css(css)->use_hierarchy;
Balbir Singh18f59ea2009-01-07 18:08:07 -08005021}
5022
Tejun Heo182446d2013-08-08 20:11:24 -04005023static int mem_cgroup_hierarchy_write(struct cgroup_subsys_state *css,
5024 struct cftype *cft, u64 val)
Balbir Singh18f59ea2009-01-07 18:08:07 -08005025{
5026 int retval = 0;
Tejun Heo182446d2013-08-08 20:11:24 -04005027 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo63876982013-08-08 20:11:23 -04005028 struct mem_cgroup *parent_memcg = mem_cgroup_from_css(css_parent(&memcg->css));
Balbir Singh18f59ea2009-01-07 18:08:07 -08005029
Glauber Costa09998212013-02-22 16:34:55 -08005030 mutex_lock(&memcg_create_mutex);
Glauber Costa567fb432012-07-31 16:43:07 -07005031
5032 if (memcg->use_hierarchy == val)
5033 goto out;
5034
Balbir Singh18f59ea2009-01-07 18:08:07 -08005035 /*
André Goddard Rosaaf901ca2009-11-14 13:09:05 -02005036 * If parent's use_hierarchy is set, we can't make any modifications
Balbir Singh18f59ea2009-01-07 18:08:07 -08005037 * in the child subtrees. If it is unset, then the change can
5038 * occur, provided the current cgroup has no children.
5039 *
5040 * For the root cgroup, parent_mem is NULL, we allow value to be
5041 * set if there are no children.
5042 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005043 if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
Balbir Singh18f59ea2009-01-07 18:08:07 -08005044 (val == 1 || val == 0)) {
Johannes Weiner696ac172013-10-31 16:34:15 -07005045 if (list_empty(&memcg->css.cgroup->children))
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005046 memcg->use_hierarchy = val;
Balbir Singh18f59ea2009-01-07 18:08:07 -08005047 else
5048 retval = -EBUSY;
5049 } else
5050 retval = -EINVAL;
Glauber Costa567fb432012-07-31 16:43:07 -07005051
5052out:
Glauber Costa09998212013-02-22 16:34:55 -08005053 mutex_unlock(&memcg_create_mutex);
Balbir Singh18f59ea2009-01-07 18:08:07 -08005054
5055 return retval;
5056}
5057
Balbir Singh0c3e73e2009-09-23 15:56:42 -07005058
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005059static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
Johannes Weiner7a159cc2011-03-23 16:42:38 -07005060 enum mem_cgroup_stat_index idx)
Balbir Singh0c3e73e2009-09-23 15:56:42 -07005061{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07005062 struct mem_cgroup *iter;
Johannes Weiner7a159cc2011-03-23 16:42:38 -07005063 long val = 0;
Balbir Singh0c3e73e2009-09-23 15:56:42 -07005064
Johannes Weiner7a159cc2011-03-23 16:42:38 -07005065 /* Per-cpu values can be negative, use a signed accumulator */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005066 for_each_mem_cgroup_tree(iter, memcg)
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07005067 val += mem_cgroup_read_stat(iter, idx);
5068
5069 if (val < 0) /* race ? */
5070 val = 0;
5071 return val;
Balbir Singh0c3e73e2009-09-23 15:56:42 -07005072}
5073
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005074static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08005075{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07005076 u64 val;
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08005077
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005078 if (!mem_cgroup_is_root(memcg)) {
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08005079 if (!swap)
Glauber Costa65c64ce2011-12-22 01:02:27 +00005080 return res_counter_read_u64(&memcg->res, RES_USAGE);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08005081 else
Glauber Costa65c64ce2011-12-22 01:02:27 +00005082 return res_counter_read_u64(&memcg->memsw, RES_USAGE);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08005083 }
5084
David Rientjesb070e652013-05-07 16:18:09 -07005085 /*
5086 * Transparent hugepages are still accounted for in MEM_CGROUP_STAT_RSS
5087 * as well as in MEM_CGROUP_STAT_RSS_HUGE.
5088 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005089 val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
5090 val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08005091
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07005092 if (swap)
Kamezawa Hiroyukibff6bb82012-07-31 16:41:38 -07005093 val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAP);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08005094
5095 return val << PAGE_SHIFT;
5096}
5097
Tejun Heo182446d2013-08-08 20:11:24 -04005098static ssize_t mem_cgroup_read(struct cgroup_subsys_state *css,
5099 struct cftype *cft, struct file *file,
5100 char __user *buf, size_t nbytes, loff_t *ppos)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005101{
Tejun Heo182446d2013-08-08 20:11:24 -04005102 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heoaf36f902012-04-01 12:09:55 -07005103 char str[64];
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08005104 u64 val;
Glauber Costa86ae53e2012-12-18 14:21:45 -08005105 int name, len;
5106 enum res_type type;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005107
5108 type = MEMFILE_TYPE(cft->private);
5109 name = MEMFILE_ATTR(cft->private);
Tejun Heoaf36f902012-04-01 12:09:55 -07005110
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005111 switch (type) {
5112 case _MEM:
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08005113 if (name == RES_USAGE)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005114 val = mem_cgroup_usage(memcg, false);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08005115 else
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005116 val = res_counter_read_u64(&memcg->res, name);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005117 break;
5118 case _MEMSWAP:
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08005119 if (name == RES_USAGE)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005120 val = mem_cgroup_usage(memcg, true);
Kirill A. Shutemov104f3922010-03-10 15:22:21 -08005121 else
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005122 val = res_counter_read_u64(&memcg->memsw, name);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005123 break;
Glauber Costa510fc4e2012-12-18 14:21:47 -08005124 case _KMEM:
5125 val = res_counter_read_u64(&memcg->kmem, name);
5126 break;
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005127 default:
5128 BUG();
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005129 }
Tejun Heoaf36f902012-04-01 12:09:55 -07005130
5131 len = scnprintf(str, sizeof(str), "%llu\n", (unsigned long long)val);
5132 return simple_read_from_buffer(buf, nbytes, ppos, str, len);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005133}
Glauber Costa510fc4e2012-12-18 14:21:47 -08005134
Tejun Heo182446d2013-08-08 20:11:24 -04005135static int memcg_update_kmem_limit(struct cgroup_subsys_state *css, u64 val)
Glauber Costa510fc4e2012-12-18 14:21:47 -08005136{
5137 int ret = -EINVAL;
5138#ifdef CONFIG_MEMCG_KMEM
Tejun Heo182446d2013-08-08 20:11:24 -04005139 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005140 /*
5141 * For simplicity, we won't allow this to be disabled. It also can't
5142 * be changed if the cgroup has children already, or if tasks had
5143 * already joined.
5144 *
5145 * If tasks join before we set the limit, a person looking at
5146 * kmem.usage_in_bytes will have no way to determine when it took
5147 * place, which makes the value quite meaningless.
5148 *
5149 * After it first became limited, changes in the value of the limit are
5150 * of course permitted.
Glauber Costa510fc4e2012-12-18 14:21:47 -08005151 */
Glauber Costa09998212013-02-22 16:34:55 -08005152 mutex_lock(&memcg_create_mutex);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005153 mutex_lock(&set_limit_mutex);
Sha Zhengju6de5a8b2013-09-12 15:13:47 -07005154 if (!memcg->kmem_account_flags && val != RES_COUNTER_MAX) {
Tejun Heo182446d2013-08-08 20:11:24 -04005155 if (cgroup_task_count(css->cgroup) || memcg_has_children(memcg)) {
Glauber Costa510fc4e2012-12-18 14:21:47 -08005156 ret = -EBUSY;
5157 goto out;
5158 }
5159 ret = res_counter_set_limit(&memcg->kmem, val);
5160 VM_BUG_ON(ret);
5161
Glauber Costa55007d82012-12-18 14:22:38 -08005162 ret = memcg_update_cache_sizes(memcg);
5163 if (ret) {
Sha Zhengju6de5a8b2013-09-12 15:13:47 -07005164 res_counter_set_limit(&memcg->kmem, RES_COUNTER_MAX);
Glauber Costa55007d82012-12-18 14:22:38 -08005165 goto out;
5166 }
Glauber Costa692e89a2013-02-22 16:34:56 -08005167 static_key_slow_inc(&memcg_kmem_enabled_key);
5168 /*
5169 * setting the active bit after the inc will guarantee no one
5170 * starts accounting before all call sites are patched
5171 */
5172 memcg_kmem_set_active(memcg);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005173 } else
5174 ret = res_counter_set_limit(&memcg->kmem, val);
5175out:
5176 mutex_unlock(&set_limit_mutex);
Glauber Costa09998212013-02-22 16:34:55 -08005177 mutex_unlock(&memcg_create_mutex);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005178#endif
5179 return ret;
5180}
5181
Hugh Dickins6d0439902013-02-22 16:35:50 -08005182#ifdef CONFIG_MEMCG_KMEM
Glauber Costa55007d82012-12-18 14:22:38 -08005183static int memcg_propagate_kmem(struct mem_cgroup *memcg)
Glauber Costa510fc4e2012-12-18 14:21:47 -08005184{
Glauber Costa55007d82012-12-18 14:22:38 -08005185 int ret = 0;
Glauber Costa510fc4e2012-12-18 14:21:47 -08005186 struct mem_cgroup *parent = parent_mem_cgroup(memcg);
5187 if (!parent)
Glauber Costa55007d82012-12-18 14:22:38 -08005188 goto out;
5189
Glauber Costa510fc4e2012-12-18 14:21:47 -08005190 memcg->kmem_account_flags = parent->kmem_account_flags;
Glauber Costaa8964b92012-12-18 14:22:09 -08005191 /*
5192 * When that happen, we need to disable the static branch only on those
5193 * memcgs that enabled it. To achieve this, we would be forced to
5194 * complicate the code by keeping track of which memcgs were the ones
5195 * that actually enabled limits, and which ones got it from its
5196 * parents.
5197 *
5198 * It is a lot simpler just to do static_key_slow_inc() on every child
5199 * that is accounted.
5200 */
Glauber Costa55007d82012-12-18 14:22:38 -08005201 if (!memcg_kmem_is_active(memcg))
5202 goto out;
5203
5204 /*
Li Zefan10d5ebf2013-07-08 16:00:33 -07005205 * __mem_cgroup_free() will issue static_key_slow_dec() because this
5206 * memcg is active already. If the later initialization fails then the
5207 * cgroup core triggers the cleanup so we do not have to do it here.
Glauber Costa55007d82012-12-18 14:22:38 -08005208 */
Glauber Costa55007d82012-12-18 14:22:38 -08005209 static_key_slow_inc(&memcg_kmem_enabled_key);
5210
5211 mutex_lock(&set_limit_mutex);
Glauber Costa425c5982013-07-08 16:00:01 -07005212 memcg_stop_kmem_account();
Glauber Costa55007d82012-12-18 14:22:38 -08005213 ret = memcg_update_cache_sizes(memcg);
Glauber Costa425c5982013-07-08 16:00:01 -07005214 memcg_resume_kmem_account();
Glauber Costa55007d82012-12-18 14:22:38 -08005215 mutex_unlock(&set_limit_mutex);
Glauber Costa55007d82012-12-18 14:22:38 -08005216out:
5217 return ret;
Glauber Costa510fc4e2012-12-18 14:21:47 -08005218}
Hugh Dickins6d0439902013-02-22 16:35:50 -08005219#endif /* CONFIG_MEMCG_KMEM */
Glauber Costa510fc4e2012-12-18 14:21:47 -08005220
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005221/*
5222 * The user of this function is...
5223 * RES_LIMIT.
5224 */
Tejun Heo182446d2013-08-08 20:11:24 -04005225static int mem_cgroup_write(struct cgroup_subsys_state *css, struct cftype *cft,
Paul Menage856c13a2008-07-25 01:47:04 -07005226 const char *buffer)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005227{
Tejun Heo182446d2013-08-08 20:11:24 -04005228 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Glauber Costa86ae53e2012-12-18 14:21:45 -08005229 enum res_type type;
5230 int name;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005231 unsigned long long val;
5232 int ret;
5233
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005234 type = MEMFILE_TYPE(cft->private);
5235 name = MEMFILE_ATTR(cft->private);
Tejun Heoaf36f902012-04-01 12:09:55 -07005236
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005237 switch (name) {
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005238 case RES_LIMIT:
Balbir Singh4b3bde42009-09-23 15:56:32 -07005239 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
5240 ret = -EINVAL;
5241 break;
5242 }
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005243 /* This function does all necessary parse...reuse it */
5244 ret = res_counter_memparse_write_strategy(buffer, &val);
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005245 if (ret)
5246 break;
5247 if (type == _MEM)
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005248 ret = mem_cgroup_resize_limit(memcg, val);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005249 else if (type == _MEMSWAP)
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005250 ret = mem_cgroup_resize_memsw_limit(memcg, val);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005251 else if (type == _KMEM)
Tejun Heo182446d2013-08-08 20:11:24 -04005252 ret = memcg_update_kmem_limit(css, val);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005253 else
5254 return -EINVAL;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005255 break;
Balbir Singh296c81d2009-09-23 15:56:36 -07005256 case RES_SOFT_LIMIT:
5257 ret = res_counter_memparse_write_strategy(buffer, &val);
5258 if (ret)
5259 break;
5260 /*
5261 * For memsw, soft limits are hard to implement in terms
5262 * of semantics, for now, we support soft limits for
5263 * control without swap
5264 */
5265 if (type == _MEM)
5266 ret = res_counter_set_soft_limit(&memcg->res, val);
5267 else
5268 ret = -EINVAL;
5269 break;
KAMEZAWA Hiroyuki628f4232008-07-25 01:47:20 -07005270 default:
5271 ret = -EINVAL; /* should be BUG() ? */
5272 break;
5273 }
5274 return ret;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005275}
5276
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005277static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
5278 unsigned long long *mem_limit, unsigned long long *memsw_limit)
5279{
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005280 unsigned long long min_limit, min_memsw_limit, tmp;
5281
5282 min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
5283 min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005284 if (!memcg->use_hierarchy)
5285 goto out;
5286
Tejun Heo63876982013-08-08 20:11:23 -04005287 while (css_parent(&memcg->css)) {
5288 memcg = mem_cgroup_from_css(css_parent(&memcg->css));
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005289 if (!memcg->use_hierarchy)
5290 break;
5291 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
5292 min_limit = min(min_limit, tmp);
5293 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
5294 min_memsw_limit = min(min_memsw_limit, tmp);
5295 }
5296out:
5297 *mem_limit = min_limit;
5298 *memsw_limit = min_memsw_limit;
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005299}
5300
Tejun Heo182446d2013-08-08 20:11:24 -04005301static int mem_cgroup_reset(struct cgroup_subsys_state *css, unsigned int event)
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07005302{
Tejun Heo182446d2013-08-08 20:11:24 -04005303 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Glauber Costa86ae53e2012-12-18 14:21:45 -08005304 int name;
5305 enum res_type type;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07005306
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005307 type = MEMFILE_TYPE(event);
5308 name = MEMFILE_ATTR(event);
Tejun Heoaf36f902012-04-01 12:09:55 -07005309
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005310 switch (name) {
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07005311 case RES_MAX_USAGE:
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005312 if (type == _MEM)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005313 res_counter_reset_max(&memcg->res);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005314 else if (type == _MEMSWAP)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005315 res_counter_reset_max(&memcg->memsw);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005316 else if (type == _KMEM)
5317 res_counter_reset_max(&memcg->kmem);
5318 else
5319 return -EINVAL;
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07005320 break;
5321 case RES_FAILCNT:
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005322 if (type == _MEM)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005323 res_counter_reset_failcnt(&memcg->res);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005324 else if (type == _MEMSWAP)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005325 res_counter_reset_failcnt(&memcg->memsw);
Glauber Costa510fc4e2012-12-18 14:21:47 -08005326 else if (type == _KMEM)
5327 res_counter_reset_failcnt(&memcg->kmem);
5328 else
5329 return -EINVAL;
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07005330 break;
5331 }
Balbir Singhf64c3f52009-09-23 15:56:37 -07005332
Pavel Emelyanov85cc59d2008-04-29 01:00:20 -07005333 return 0;
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07005334}
5335
Tejun Heo182446d2013-08-08 20:11:24 -04005336static u64 mem_cgroup_move_charge_read(struct cgroup_subsys_state *css,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005337 struct cftype *cft)
5338{
Tejun Heo182446d2013-08-08 20:11:24 -04005339 return mem_cgroup_from_css(css)->move_charge_at_immigrate;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005340}
5341
Daisuke Nishimura02491442010-03-10 15:22:17 -08005342#ifdef CONFIG_MMU
Tejun Heo182446d2013-08-08 20:11:24 -04005343static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005344 struct cftype *cft, u64 val)
5345{
Tejun Heo182446d2013-08-08 20:11:24 -04005346 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005347
5348 if (val >= (1 << NR_MOVE_TYPE))
5349 return -EINVAL;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005350
Glauber Costaee5e8472013-02-22 16:34:50 -08005351 /*
5352 * No kind of locking is needed in here, because ->can_attach() will
5353 * check this value once in the beginning of the process, and then carry
5354 * on with stale data. This means that changes to this value will only
5355 * affect task migrations starting after the change.
5356 */
5357 memcg->move_charge_at_immigrate = val;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005358 return 0;
5359}
Daisuke Nishimura02491442010-03-10 15:22:17 -08005360#else
Tejun Heo182446d2013-08-08 20:11:24 -04005361static int mem_cgroup_move_charge_write(struct cgroup_subsys_state *css,
Daisuke Nishimura02491442010-03-10 15:22:17 -08005362 struct cftype *cft, u64 val)
5363{
5364 return -ENOSYS;
5365}
5366#endif
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005367
Ying Han406eb0c2011-05-26 16:25:37 -07005368#ifdef CONFIG_NUMA
Tejun Heo182446d2013-08-08 20:11:24 -04005369static int memcg_numa_stat_show(struct cgroup_subsys_state *css,
5370 struct cftype *cft, struct seq_file *m)
Ying Han406eb0c2011-05-26 16:25:37 -07005371{
5372 int nid;
5373 unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
5374 unsigned long node_nr;
Tejun Heo182446d2013-08-08 20:11:24 -04005375 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Ying Han406eb0c2011-05-26 16:25:37 -07005376
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005377 total_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL);
Ying Han406eb0c2011-05-26 16:25:37 -07005378 seq_printf(m, "total=%lu", total_nr);
Lai Jiangshan31aaea42012-12-12 13:51:27 -08005379 for_each_node_state(nid, N_MEMORY) {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005380 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL);
Ying Han406eb0c2011-05-26 16:25:37 -07005381 seq_printf(m, " N%d=%lu", nid, node_nr);
5382 }
5383 seq_putc(m, '\n');
5384
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005385 file_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_FILE);
Ying Han406eb0c2011-05-26 16:25:37 -07005386 seq_printf(m, "file=%lu", file_nr);
Lai Jiangshan31aaea42012-12-12 13:51:27 -08005387 for_each_node_state(nid, N_MEMORY) {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005388 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07005389 LRU_ALL_FILE);
Ying Han406eb0c2011-05-26 16:25:37 -07005390 seq_printf(m, " N%d=%lu", nid, node_nr);
5391 }
5392 seq_putc(m, '\n');
5393
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005394 anon_nr = mem_cgroup_nr_lru_pages(memcg, LRU_ALL_ANON);
Ying Han406eb0c2011-05-26 16:25:37 -07005395 seq_printf(m, "anon=%lu", anon_nr);
Lai Jiangshan31aaea42012-12-12 13:51:27 -08005396 for_each_node_state(nid, N_MEMORY) {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005397 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07005398 LRU_ALL_ANON);
Ying Han406eb0c2011-05-26 16:25:37 -07005399 seq_printf(m, " N%d=%lu", nid, node_nr);
5400 }
5401 seq_putc(m, '\n');
5402
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005403 unevictable_nr = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
Ying Han406eb0c2011-05-26 16:25:37 -07005404 seq_printf(m, "unevictable=%lu", unevictable_nr);
Lai Jiangshan31aaea42012-12-12 13:51:27 -08005405 for_each_node_state(nid, N_MEMORY) {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005406 node_nr = mem_cgroup_node_nr_lru_pages(memcg, nid,
KAMEZAWA Hiroyukibb2a0de2011-07-26 16:08:22 -07005407 BIT(LRU_UNEVICTABLE));
Ying Han406eb0c2011-05-26 16:25:37 -07005408 seq_printf(m, " N%d=%lu", nid, node_nr);
5409 }
5410 seq_putc(m, '\n');
5411 return 0;
5412}
5413#endif /* CONFIG_NUMA */
5414
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005415static inline void mem_cgroup_lru_names_not_uptodate(void)
5416{
5417 BUILD_BUG_ON(ARRAY_SIZE(mem_cgroup_lru_names) != NR_LRU_LISTS);
5418}
5419
Tejun Heo182446d2013-08-08 20:11:24 -04005420static int memcg_stat_show(struct cgroup_subsys_state *css, struct cftype *cft,
Johannes Weiner78ccf5b2012-05-29 15:07:06 -07005421 struct seq_file *m)
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08005422{
Tejun Heo182446d2013-08-08 20:11:24 -04005423 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005424 struct mem_cgroup *mi;
5425 unsigned int i;
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08005426
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005427 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
Kamezawa Hiroyukibff6bb82012-07-31 16:41:38 -07005428 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07005429 continue;
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005430 seq_printf(m, "%s %ld\n", mem_cgroup_stat_names[i],
5431 mem_cgroup_read_stat(memcg, i) * PAGE_SIZE);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07005432 }
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08005433
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005434 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++)
5435 seq_printf(m, "%s %lu\n", mem_cgroup_events_names[i],
5436 mem_cgroup_read_events(memcg, i));
5437
5438 for (i = 0; i < NR_LRU_LISTS; i++)
5439 seq_printf(m, "%s %lu\n", mem_cgroup_lru_names[i],
5440 mem_cgroup_nr_lru_pages(memcg, BIT(i)) * PAGE_SIZE);
5441
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07005442 /* Hierarchical information */
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005443 {
5444 unsigned long long limit, memsw_limit;
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005445 memcg_get_hierarchical_limit(memcg, &limit, &memsw_limit);
Johannes Weiner78ccf5b2012-05-29 15:07:06 -07005446 seq_printf(m, "hierarchical_memory_limit %llu\n", limit);
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005447 if (do_swap_account)
Johannes Weiner78ccf5b2012-05-29 15:07:06 -07005448 seq_printf(m, "hierarchical_memsw_limit %llu\n",
5449 memsw_limit);
KAMEZAWA Hiroyukifee7b542009-01-07 18:08:26 -08005450 }
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005451
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005452 for (i = 0; i < MEM_CGROUP_STAT_NSTATS; i++) {
5453 long long val = 0;
5454
Kamezawa Hiroyukibff6bb82012-07-31 16:41:38 -07005455 if (i == MEM_CGROUP_STAT_SWAP && !do_swap_account)
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07005456 continue;
Johannes Weineraf7c4b02012-05-29 15:07:08 -07005457 for_each_mem_cgroup_tree(mi, memcg)
5458 val += mem_cgroup_read_stat(mi, i) * PAGE_SIZE;
5459 seq_printf(m, "total_%s %lld\n", mem_cgroup_stat_names[i], val);
5460 }
5461
5462 for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
5463 unsigned long long val = 0;
5464
5465 for_each_mem_cgroup_tree(mi, memcg)
5466 val += mem_cgroup_read_events(mi, i);
5467 seq_printf(m, "total_%s %llu\n",
5468 mem_cgroup_events_names[i], val);
5469 }
5470
5471 for (i = 0; i < NR_LRU_LISTS; i++) {
5472 unsigned long long val = 0;
5473
5474 for_each_mem_cgroup_tree(mi, memcg)
5475 val += mem_cgroup_nr_lru_pages(mi, BIT(i)) * PAGE_SIZE;
5476 seq_printf(m, "total_%s %llu\n", mem_cgroup_lru_names[i], val);
Daisuke Nishimura1dd3a272009-09-23 15:56:43 -07005477 }
KAMEZAWA Hiroyuki14067bb2009-04-02 16:57:35 -07005478
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005479#ifdef CONFIG_DEBUG_VM
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005480 {
5481 int nid, zid;
5482 struct mem_cgroup_per_zone *mz;
Hugh Dickins89abfab2012-05-29 15:06:53 -07005483 struct zone_reclaim_stat *rstat;
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005484 unsigned long recent_rotated[2] = {0, 0};
5485 unsigned long recent_scanned[2] = {0, 0};
5486
5487 for_each_online_node(nid)
5488 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
Hugh Dickinsd79154b2012-03-21 16:34:18 -07005489 mz = mem_cgroup_zoneinfo(memcg, nid, zid);
Hugh Dickins89abfab2012-05-29 15:06:53 -07005490 rstat = &mz->lruvec.reclaim_stat;
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005491
Hugh Dickins89abfab2012-05-29 15:06:53 -07005492 recent_rotated[0] += rstat->recent_rotated[0];
5493 recent_rotated[1] += rstat->recent_rotated[1];
5494 recent_scanned[0] += rstat->recent_scanned[0];
5495 recent_scanned[1] += rstat->recent_scanned[1];
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005496 }
Johannes Weiner78ccf5b2012-05-29 15:07:06 -07005497 seq_printf(m, "recent_rotated_anon %lu\n", recent_rotated[0]);
5498 seq_printf(m, "recent_rotated_file %lu\n", recent_rotated[1]);
5499 seq_printf(m, "recent_scanned_anon %lu\n", recent_scanned[0]);
5500 seq_printf(m, "recent_scanned_file %lu\n", recent_scanned[1]);
KOSAKI Motohiro7f016ee2009-01-07 18:08:22 -08005501 }
5502#endif
5503
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08005504 return 0;
5505}
5506
Tejun Heo182446d2013-08-08 20:11:24 -04005507static u64 mem_cgroup_swappiness_read(struct cgroup_subsys_state *css,
5508 struct cftype *cft)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005509{
Tejun Heo182446d2013-08-08 20:11:24 -04005510 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005511
KAMEZAWA Hiroyuki1f4c0252011-07-26 16:08:21 -07005512 return mem_cgroup_swappiness(memcg);
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005513}
5514
Tejun Heo182446d2013-08-08 20:11:24 -04005515static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
5516 struct cftype *cft, u64 val)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005517{
Tejun Heo182446d2013-08-08 20:11:24 -04005518 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo63876982013-08-08 20:11:23 -04005519 struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(&memcg->css));
Li Zefan068b38c2009-01-15 13:51:26 -08005520
Tejun Heo63876982013-08-08 20:11:23 -04005521 if (val > 100 || !parent)
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005522 return -EINVAL;
5523
Glauber Costa09998212013-02-22 16:34:55 -08005524 mutex_lock(&memcg_create_mutex);
Li Zefan068b38c2009-01-15 13:51:26 -08005525
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005526 /* If under hierarchy, only empty-root can set this value */
Glauber Costab5f99b52013-02-22 16:34:53 -08005527 if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
Glauber Costa09998212013-02-22 16:34:55 -08005528 mutex_unlock(&memcg_create_mutex);
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005529 return -EINVAL;
Li Zefan068b38c2009-01-15 13:51:26 -08005530 }
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005531
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005532 memcg->swappiness = val;
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005533
Glauber Costa09998212013-02-22 16:34:55 -08005534 mutex_unlock(&memcg_create_mutex);
Li Zefan068b38c2009-01-15 13:51:26 -08005535
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005536 return 0;
5537}
5538
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005539static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
5540{
5541 struct mem_cgroup_threshold_ary *t;
5542 u64 usage;
5543 int i;
5544
5545 rcu_read_lock();
5546 if (!swap)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005547 t = rcu_dereference(memcg->thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005548 else
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005549 t = rcu_dereference(memcg->memsw_thresholds.primary);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005550
5551 if (!t)
5552 goto unlock;
5553
5554 usage = mem_cgroup_usage(memcg, swap);
5555
5556 /*
Sha Zhengju748dad32012-05-29 15:06:57 -07005557 * current_threshold points to threshold just below or equal to usage.
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005558 * If it's not true, a threshold was crossed after last
5559 * call of __mem_cgroup_threshold().
5560 */
Phil Carmody5407a562010-05-26 14:42:42 -07005561 i = t->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005562
5563 /*
5564 * Iterate backward over array of thresholds starting from
5565 * current_threshold and check if a threshold is crossed.
5566 * If none of thresholds below usage is crossed, we read
5567 * only one element of the array here.
5568 */
5569 for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
5570 eventfd_signal(t->entries[i].eventfd, 1);
5571
5572 /* i = current_threshold + 1 */
5573 i++;
5574
5575 /*
5576 * Iterate forward over array of thresholds starting from
5577 * current_threshold+1 and check if a threshold is crossed.
5578 * If none of thresholds above usage is crossed, we read
5579 * only one element of the array here.
5580 */
5581 for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
5582 eventfd_signal(t->entries[i].eventfd, 1);
5583
5584 /* Update current_threshold */
Phil Carmody5407a562010-05-26 14:42:42 -07005585 t->current_threshold = i - 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005586unlock:
5587 rcu_read_unlock();
5588}
5589
5590static void mem_cgroup_threshold(struct mem_cgroup *memcg)
5591{
Kirill A. Shutemovad4ca5f2010-10-07 12:59:27 -07005592 while (memcg) {
5593 __mem_cgroup_threshold(memcg, false);
5594 if (do_swap_account)
5595 __mem_cgroup_threshold(memcg, true);
5596
5597 memcg = parent_mem_cgroup(memcg);
5598 }
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005599}
5600
5601static int compare_thresholds(const void *a, const void *b)
5602{
5603 const struct mem_cgroup_threshold *_a = a;
5604 const struct mem_cgroup_threshold *_b = b;
5605
Greg Thelen2bff24a2013-09-11 14:23:08 -07005606 if (_a->threshold > _b->threshold)
5607 return 1;
5608
5609 if (_a->threshold < _b->threshold)
5610 return -1;
5611
5612 return 0;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005613}
5614
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005615static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005616{
5617 struct mem_cgroup_eventfd_list *ev;
5618
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005619 list_for_each_entry(ev, &memcg->oom_notify, list)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005620 eventfd_signal(ev->eventfd, 1);
5621 return 0;
5622}
5623
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005624static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005625{
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07005626 struct mem_cgroup *iter;
5627
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005628 for_each_mem_cgroup_tree(iter, memcg)
KAMEZAWA Hiroyuki7d74b062010-10-27 15:33:41 -07005629 mem_cgroup_oom_notify_cb(iter);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005630}
5631
Tejun Heo81eeaf02013-08-08 20:11:26 -04005632static int mem_cgroup_usage_register_event(struct cgroup_subsys_state *css,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005633 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005634{
Tejun Heo81eeaf02013-08-08 20:11:26 -04005635 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005636 struct mem_cgroup_thresholds *thresholds;
5637 struct mem_cgroup_threshold_ary *new;
Glauber Costa86ae53e2012-12-18 14:21:45 -08005638 enum res_type type = MEMFILE_TYPE(cft->private);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005639 u64 threshold, usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005640 int i, size, ret;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005641
5642 ret = res_counter_memparse_write_strategy(args, &threshold);
5643 if (ret)
5644 return ret;
5645
5646 mutex_lock(&memcg->thresholds_lock);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005647
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005648 if (type == _MEM)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005649 thresholds = &memcg->thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005650 else if (type == _MEMSWAP)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005651 thresholds = &memcg->memsw_thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005652 else
5653 BUG();
5654
5655 usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
5656
5657 /* Check if a threshold crossed before adding a new one */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005658 if (thresholds->primary)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005659 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
5660
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005661 size = thresholds->primary ? thresholds->primary->size + 1 : 1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005662
5663 /* Allocate memory for new array of thresholds */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005664 new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005665 GFP_KERNEL);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005666 if (!new) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005667 ret = -ENOMEM;
5668 goto unlock;
5669 }
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005670 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005671
5672 /* Copy thresholds (if any) to new array */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005673 if (thresholds->primary) {
5674 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005675 sizeof(struct mem_cgroup_threshold));
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005676 }
5677
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005678 /* Add new threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005679 new->entries[size - 1].eventfd = eventfd;
5680 new->entries[size - 1].threshold = threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005681
5682 /* Sort thresholds. Registering of new threshold isn't time-critical */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005683 sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005684 compare_thresholds, NULL);
5685
5686 /* Find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005687 new->current_threshold = -1;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005688 for (i = 0; i < size; i++) {
Sha Zhengju748dad32012-05-29 15:06:57 -07005689 if (new->entries[i].threshold <= usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005690 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005691 * new->current_threshold will not be used until
5692 * rcu_assign_pointer(), so it's safe to increment
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005693 * it here.
5694 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005695 ++new->current_threshold;
Sha Zhengju748dad32012-05-29 15:06:57 -07005696 } else
5697 break;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005698 }
5699
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005700 /* Free old spare buffer and save old primary buffer as spare */
5701 kfree(thresholds->spare);
5702 thresholds->spare = thresholds->primary;
5703
5704 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005705
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07005706 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005707 synchronize_rcu();
5708
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005709unlock:
5710 mutex_unlock(&memcg->thresholds_lock);
5711
5712 return ret;
5713}
5714
Tejun Heo81eeaf02013-08-08 20:11:26 -04005715static void mem_cgroup_usage_unregister_event(struct cgroup_subsys_state *css,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005716 struct cftype *cft, struct eventfd_ctx *eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005717{
Tejun Heo81eeaf02013-08-08 20:11:26 -04005718 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005719 struct mem_cgroup_thresholds *thresholds;
5720 struct mem_cgroup_threshold_ary *new;
Glauber Costa86ae53e2012-12-18 14:21:45 -08005721 enum res_type type = MEMFILE_TYPE(cft->private);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005722 u64 usage;
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005723 int i, j, size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005724
5725 mutex_lock(&memcg->thresholds_lock);
5726 if (type == _MEM)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005727 thresholds = &memcg->thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005728 else if (type == _MEMSWAP)
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005729 thresholds = &memcg->memsw_thresholds;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005730 else
5731 BUG();
5732
Anton Vorontsov371528c2012-02-24 05:14:46 +04005733 if (!thresholds->primary)
5734 goto unlock;
5735
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005736 usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
5737
5738 /* Check if a threshold crossed before removing */
5739 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
5740
5741 /* Calculate new number of threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005742 size = 0;
5743 for (i = 0; i < thresholds->primary->size; i++) {
5744 if (thresholds->primary->entries[i].eventfd != eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005745 size++;
5746 }
5747
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005748 new = thresholds->spare;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07005749
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005750 /* Set thresholds array to NULL if we don't have thresholds */
5751 if (!size) {
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005752 kfree(new);
5753 new = NULL;
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07005754 goto swap_buffers;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005755 }
5756
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005757 new->size = size;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005758
5759 /* Copy thresholds and find current threshold */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005760 new->current_threshold = -1;
5761 for (i = 0, j = 0; i < thresholds->primary->size; i++) {
5762 if (thresholds->primary->entries[i].eventfd == eventfd)
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005763 continue;
5764
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005765 new->entries[j] = thresholds->primary->entries[i];
Sha Zhengju748dad32012-05-29 15:06:57 -07005766 if (new->entries[j].threshold <= usage) {
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005767 /*
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005768 * new->current_threshold will not be used
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005769 * until rcu_assign_pointer(), so it's safe to increment
5770 * it here.
5771 */
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005772 ++new->current_threshold;
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005773 }
5774 j++;
5775 }
5776
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07005777swap_buffers:
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005778 /* Swap primary and spare array */
5779 thresholds->spare = thresholds->primary;
Sha Zhengju8c757762012-05-10 13:01:45 -07005780 /* If all events are unregistered, free the spare array */
5781 if (!new) {
5782 kfree(thresholds->spare);
5783 thresholds->spare = NULL;
5784 }
5785
Kirill A. Shutemov2c488db2010-05-26 14:42:47 -07005786 rcu_assign_pointer(thresholds->primary, new);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005787
Kirill A. Shutemov907860e2010-05-26 14:42:46 -07005788 /* To be sure that nobody uses thresholds */
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005789 synchronize_rcu();
Anton Vorontsov371528c2012-02-24 05:14:46 +04005790unlock:
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005791 mutex_unlock(&memcg->thresholds_lock);
Kirill A. Shutemov2e72b632010-03-10 15:22:24 -08005792}
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08005793
Tejun Heo81eeaf02013-08-08 20:11:26 -04005794static int mem_cgroup_oom_register_event(struct cgroup_subsys_state *css,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005795 struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
5796{
Tejun Heo81eeaf02013-08-08 20:11:26 -04005797 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005798 struct mem_cgroup_eventfd_list *event;
Glauber Costa86ae53e2012-12-18 14:21:45 -08005799 enum res_type type = MEMFILE_TYPE(cft->private);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005800
5801 BUG_ON(type != _OOM_TYPE);
5802 event = kmalloc(sizeof(*event), GFP_KERNEL);
5803 if (!event)
5804 return -ENOMEM;
5805
Michal Hocko1af8efe2011-07-26 16:08:24 -07005806 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005807
5808 event->eventfd = eventfd;
5809 list_add(&event->list, &memcg->oom_notify);
5810
5811 /* already in OOM ? */
Michal Hocko79dfdac2011-07-26 16:08:23 -07005812 if (atomic_read(&memcg->under_oom))
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005813 eventfd_signal(eventfd, 1);
Michal Hocko1af8efe2011-07-26 16:08:24 -07005814 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005815
5816 return 0;
5817}
5818
Tejun Heo81eeaf02013-08-08 20:11:26 -04005819static void mem_cgroup_oom_unregister_event(struct cgroup_subsys_state *css,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005820 struct cftype *cft, struct eventfd_ctx *eventfd)
5821{
Tejun Heo81eeaf02013-08-08 20:11:26 -04005822 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005823 struct mem_cgroup_eventfd_list *ev, *tmp;
Glauber Costa86ae53e2012-12-18 14:21:45 -08005824 enum res_type type = MEMFILE_TYPE(cft->private);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005825
5826 BUG_ON(type != _OOM_TYPE);
5827
Michal Hocko1af8efe2011-07-26 16:08:24 -07005828 spin_lock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005829
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005830 list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005831 if (ev->eventfd == eventfd) {
5832 list_del(&ev->list);
5833 kfree(ev);
5834 }
5835 }
5836
Michal Hocko1af8efe2011-07-26 16:08:24 -07005837 spin_unlock(&memcg_oom_lock);
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005838}
5839
Tejun Heo182446d2013-08-08 20:11:24 -04005840static int mem_cgroup_oom_control_read(struct cgroup_subsys_state *css,
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005841 struct cftype *cft, struct cgroup_map_cb *cb)
5842{
Tejun Heo182446d2013-08-08 20:11:24 -04005843 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005844
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005845 cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005846
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005847 if (atomic_read(&memcg->under_oom))
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005848 cb->fill(cb, "under_oom", 1);
5849 else
5850 cb->fill(cb, "under_oom", 0);
5851 return 0;
5852}
5853
Tejun Heo182446d2013-08-08 20:11:24 -04005854static int mem_cgroup_oom_control_write(struct cgroup_subsys_state *css,
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005855 struct cftype *cft, u64 val)
5856{
Tejun Heo182446d2013-08-08 20:11:24 -04005857 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Tejun Heo63876982013-08-08 20:11:23 -04005858 struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(&memcg->css));
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005859
5860 /* cannot set to root cgroup and only 0 and 1 are allowed */
Tejun Heo63876982013-08-08 20:11:23 -04005861 if (!parent || !((val == 0) || (val == 1)))
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005862 return -EINVAL;
5863
Glauber Costa09998212013-02-22 16:34:55 -08005864 mutex_lock(&memcg_create_mutex);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005865 /* oom-kill-disable is a flag for subhierarchy. */
Glauber Costab5f99b52013-02-22 16:34:53 -08005866 if ((parent->use_hierarchy) || memcg_has_children(memcg)) {
Glauber Costa09998212013-02-22 16:34:55 -08005867 mutex_unlock(&memcg_create_mutex);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005868 return -EINVAL;
5869 }
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005870 memcg->oom_kill_disable = val;
KAMEZAWA Hiroyuki4d845eb2010-06-29 15:05:18 -07005871 if (!val)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07005872 memcg_oom_recover(memcg);
Glauber Costa09998212013-02-22 16:34:55 -08005873 mutex_unlock(&memcg_create_mutex);
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07005874 return 0;
5875}
5876
Andrew Mortonc255a452012-07-31 16:43:02 -07005877#ifdef CONFIG_MEMCG_KMEM
Glauber Costacbe128e32012-04-09 19:36:34 -03005878static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
Glauber Costae5671df2011-12-11 21:47:01 +00005879{
Glauber Costa55007d82012-12-18 14:22:38 -08005880 int ret;
5881
Glauber Costa2633d7a2012-12-18 14:22:34 -08005882 memcg->kmemcg_id = -1;
Glauber Costa55007d82012-12-18 14:22:38 -08005883 ret = memcg_propagate_kmem(memcg);
5884 if (ret)
5885 return ret;
Glauber Costa2633d7a2012-12-18 14:22:34 -08005886
Glauber Costa1d62e432012-04-09 19:36:33 -03005887 return mem_cgroup_sockets_init(memcg, ss);
Michel Lespinasse573b4002013-04-29 15:08:13 -07005888}
Glauber Costae5671df2011-12-11 21:47:01 +00005889
Li Zefan10d5ebf2013-07-08 16:00:33 -07005890static void memcg_destroy_kmem(struct mem_cgroup *memcg)
Glauber Costad1a4c0b2011-12-11 21:47:04 +00005891{
Glauber Costa1d62e432012-04-09 19:36:33 -03005892 mem_cgroup_sockets_destroy(memcg);
Li Zefan10d5ebf2013-07-08 16:00:33 -07005893}
5894
5895static void kmem_cgroup_css_offline(struct mem_cgroup *memcg)
5896{
5897 if (!memcg_kmem_is_active(memcg))
5898 return;
5899
5900 /*
5901 * kmem charges can outlive the cgroup. In the case of slab
5902 * pages, for instance, a page contain objects from various
5903 * processes. As we prevent from taking a reference for every
5904 * such allocation we have to be careful when doing uncharge
5905 * (see memcg_uncharge_kmem) and here during offlining.
5906 *
5907 * The idea is that that only the _last_ uncharge which sees
5908 * the dead memcg will drop the last reference. An additional
5909 * reference is taken here before the group is marked dead
5910 * which is then paired with css_put during uncharge resp. here.
5911 *
5912 * Although this might sound strange as this path is called from
5913 * css_offline() when the referencemight have dropped down to 0
5914 * and shouldn't be incremented anymore (css_tryget would fail)
5915 * we do not have other options because of the kmem allocations
5916 * lifetime.
5917 */
5918 css_get(&memcg->css);
Glauber Costa7de37682012-12-18 14:22:07 -08005919
5920 memcg_kmem_mark_dead(memcg);
5921
5922 if (res_counter_read_u64(&memcg->kmem, RES_USAGE) != 0)
5923 return;
5924
Glauber Costa7de37682012-12-18 14:22:07 -08005925 if (memcg_kmem_test_and_clear_dead(memcg))
Li Zefan10d5ebf2013-07-08 16:00:33 -07005926 css_put(&memcg->css);
Glauber Costad1a4c0b2011-12-11 21:47:04 +00005927}
Glauber Costae5671df2011-12-11 21:47:01 +00005928#else
Glauber Costacbe128e32012-04-09 19:36:34 -03005929static int memcg_init_kmem(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
Glauber Costae5671df2011-12-11 21:47:01 +00005930{
5931 return 0;
5932}
Glauber Costad1a4c0b2011-12-11 21:47:04 +00005933
Li Zefan10d5ebf2013-07-08 16:00:33 -07005934static void memcg_destroy_kmem(struct mem_cgroup *memcg)
5935{
5936}
5937
5938static void kmem_cgroup_css_offline(struct mem_cgroup *memcg)
Glauber Costad1a4c0b2011-12-11 21:47:04 +00005939{
5940}
Glauber Costae5671df2011-12-11 21:47:01 +00005941#endif
5942
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005943static struct cftype mem_cgroup_files[] = {
5944 {
Balbir Singh0eea1032008-02-07 00:13:57 -08005945 .name = "usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005946 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
Tejun Heoaf36f902012-04-01 12:09:55 -07005947 .read = mem_cgroup_read,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005948 .register_event = mem_cgroup_usage_register_event,
5949 .unregister_event = mem_cgroup_usage_unregister_event,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005950 },
5951 {
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07005952 .name = "max_usage_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005953 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07005954 .trigger = mem_cgroup_reset,
Tejun Heoaf36f902012-04-01 12:09:55 -07005955 .read = mem_cgroup_read,
Pavel Emelyanovc84872e2008-04-29 01:00:17 -07005956 },
5957 {
Balbir Singh0eea1032008-02-07 00:13:57 -08005958 .name = "limit_in_bytes",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005959 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
Paul Menage856c13a2008-07-25 01:47:04 -07005960 .write_string = mem_cgroup_write,
Tejun Heoaf36f902012-04-01 12:09:55 -07005961 .read = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005962 },
5963 {
Balbir Singh296c81d2009-09-23 15:56:36 -07005964 .name = "soft_limit_in_bytes",
5965 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
5966 .write_string = mem_cgroup_write,
Tejun Heoaf36f902012-04-01 12:09:55 -07005967 .read = mem_cgroup_read,
Balbir Singh296c81d2009-09-23 15:56:36 -07005968 },
5969 {
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005970 .name = "failcnt",
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08005971 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
Pavel Emelyanov29f2a4d2008-04-29 01:00:21 -07005972 .trigger = mem_cgroup_reset,
Tejun Heoaf36f902012-04-01 12:09:55 -07005973 .read = mem_cgroup_read,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08005974 },
Balbir Singh8697d332008-02-07 00:13:59 -08005975 {
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08005976 .name = "stat",
Wanpeng Liab215882012-07-31 16:43:09 -07005977 .read_seq_string = memcg_stat_show,
KAMEZAWA Hiroyukid2ceb9b2008-02-07 00:14:25 -08005978 },
KAMEZAWA Hiroyukic1e862c2009-01-07 18:07:55 -08005979 {
5980 .name = "force_empty",
5981 .trigger = mem_cgroup_force_empty_write,
5982 },
Balbir Singh18f59ea2009-01-07 18:08:07 -08005983 {
5984 .name = "use_hierarchy",
Tejun Heof00baae2013-04-15 13:41:15 -07005985 .flags = CFTYPE_INSANE,
Balbir Singh18f59ea2009-01-07 18:08:07 -08005986 .write_u64 = mem_cgroup_hierarchy_write,
5987 .read_u64 = mem_cgroup_hierarchy_read,
5988 },
KOSAKI Motohiroa7885eb2009-01-07 18:08:24 -08005989 {
5990 .name = "swappiness",
5991 .read_u64 = mem_cgroup_swappiness_read,
5992 .write_u64 = mem_cgroup_swappiness_write,
5993 },
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08005994 {
5995 .name = "move_charge_at_immigrate",
5996 .read_u64 = mem_cgroup_move_charge_read,
5997 .write_u64 = mem_cgroup_move_charge_write,
5998 },
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07005999 {
6000 .name = "oom_control",
KAMEZAWA Hiroyuki3c11ecf2010-05-26 14:42:37 -07006001 .read_map = mem_cgroup_oom_control_read,
6002 .write_u64 = mem_cgroup_oom_control_write,
KAMEZAWA Hiroyuki9490ff22010-05-26 14:42:36 -07006003 .register_event = mem_cgroup_oom_register_event,
6004 .unregister_event = mem_cgroup_oom_unregister_event,
6005 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
6006 },
Anton Vorontsov70ddf632013-04-29 15:08:31 -07006007 {
6008 .name = "pressure_level",
6009 .register_event = vmpressure_register_event,
6010 .unregister_event = vmpressure_unregister_event,
6011 },
Ying Han406eb0c2011-05-26 16:25:37 -07006012#ifdef CONFIG_NUMA
6013 {
6014 .name = "numa_stat",
Wanpeng Liab215882012-07-31 16:43:09 -07006015 .read_seq_string = memcg_numa_stat_show,
Ying Han406eb0c2011-05-26 16:25:37 -07006016 },
6017#endif
Glauber Costa510fc4e2012-12-18 14:21:47 -08006018#ifdef CONFIG_MEMCG_KMEM
6019 {
6020 .name = "kmem.limit_in_bytes",
6021 .private = MEMFILE_PRIVATE(_KMEM, RES_LIMIT),
6022 .write_string = mem_cgroup_write,
6023 .read = mem_cgroup_read,
6024 },
6025 {
6026 .name = "kmem.usage_in_bytes",
6027 .private = MEMFILE_PRIVATE(_KMEM, RES_USAGE),
6028 .read = mem_cgroup_read,
6029 },
6030 {
6031 .name = "kmem.failcnt",
6032 .private = MEMFILE_PRIVATE(_KMEM, RES_FAILCNT),
6033 .trigger = mem_cgroup_reset,
6034 .read = mem_cgroup_read,
6035 },
6036 {
6037 .name = "kmem.max_usage_in_bytes",
6038 .private = MEMFILE_PRIVATE(_KMEM, RES_MAX_USAGE),
6039 .trigger = mem_cgroup_reset,
6040 .read = mem_cgroup_read,
6041 },
Glauber Costa749c5412012-12-18 14:23:01 -08006042#ifdef CONFIG_SLABINFO
6043 {
6044 .name = "kmem.slabinfo",
6045 .read_seq_string = mem_cgroup_slabinfo_read,
6046 },
6047#endif
Glauber Costa510fc4e2012-12-18 14:21:47 -08006048#endif
Tejun Heo6bc10342012-04-01 12:09:55 -07006049 { }, /* terminate */
Tejun Heoaf36f902012-04-01 12:09:55 -07006050};
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08006051
Michal Hocko2d110852013-02-22 16:34:43 -08006052#ifdef CONFIG_MEMCG_SWAP
6053static struct cftype memsw_cgroup_files[] = {
6054 {
6055 .name = "memsw.usage_in_bytes",
6056 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
6057 .read = mem_cgroup_read,
6058 .register_event = mem_cgroup_usage_register_event,
6059 .unregister_event = mem_cgroup_usage_unregister_event,
6060 },
6061 {
6062 .name = "memsw.max_usage_in_bytes",
6063 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
6064 .trigger = mem_cgroup_reset,
6065 .read = mem_cgroup_read,
6066 },
6067 {
6068 .name = "memsw.limit_in_bytes",
6069 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
6070 .write_string = mem_cgroup_write,
6071 .read = mem_cgroup_read,
6072 },
6073 {
6074 .name = "memsw.failcnt",
6075 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
6076 .trigger = mem_cgroup_reset,
6077 .read = mem_cgroup_read,
6078 },
6079 { }, /* terminate */
6080};
6081#endif
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006082static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006083{
6084 struct mem_cgroup_per_node *pn;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08006085 struct mem_cgroup_per_zone *mz;
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07006086 int zone, tmp = node;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08006087 /*
6088 * This routine is called against possible nodes.
6089 * But it's BUG to call kmalloc() against offline node.
6090 *
6091 * TODO: this routine can waste much memory for nodes which will
6092 * never be onlined. It's better to use memory hotplug callback
6093 * function.
6094 */
KAMEZAWA Hiroyuki41e33552008-04-08 17:41:54 -07006095 if (!node_state(node, N_NORMAL_MEMORY))
6096 tmp = -1;
Jesper Juhl17295c82011-01-13 15:47:42 -08006097 pn = kzalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006098 if (!pn)
6099 return 1;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08006100
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08006101 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
6102 mz = &pn->zoneinfo[zone];
Hugh Dickinsbea8c152012-11-16 14:14:54 -08006103 lruvec_init(&mz->lruvec);
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -07006104 mz->usage_in_excess = 0;
6105 mz->on_tree = false;
Hugh Dickinsd79154b2012-03-21 16:34:18 -07006106 mz->memcg = memcg;
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08006107 }
Johannes Weiner54f72fe2013-07-08 15:59:49 -07006108 memcg->nodeinfo[node] = pn;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006109 return 0;
6110}
6111
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006112static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08006113{
Johannes Weiner54f72fe2013-07-08 15:59:49 -07006114 kfree(memcg->nodeinfo[node]);
KAMEZAWA Hiroyuki1ecaab22008-02-07 00:14:38 -08006115}
6116
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07006117static struct mem_cgroup *mem_cgroup_alloc(void)
6118{
Hugh Dickinsd79154b2012-03-21 16:34:18 -07006119 struct mem_cgroup *memcg;
Glauber Costa45cf7eb2013-02-22 16:34:49 -08006120 size_t size = memcg_size();
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07006121
Glauber Costa45cf7eb2013-02-22 16:34:49 -08006122 /* Can be very big if nr_node_ids is very big */
Jan Blunckc8dad2b2009-01-07 18:07:53 -08006123 if (size < PAGE_SIZE)
Hugh Dickinsd79154b2012-03-21 16:34:18 -07006124 memcg = kzalloc(size, GFP_KERNEL);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07006125 else
Hugh Dickinsd79154b2012-03-21 16:34:18 -07006126 memcg = vzalloc(size);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07006127
Hugh Dickinsd79154b2012-03-21 16:34:18 -07006128 if (!memcg)
Dan Carpentere7bbcdf2010-03-23 13:35:12 -07006129 return NULL;
6130
Hugh Dickinsd79154b2012-03-21 16:34:18 -07006131 memcg->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
6132 if (!memcg->stat)
Dan Carpenterd2e61b82010-11-11 14:05:12 -08006133 goto out_free;
Hugh Dickinsd79154b2012-03-21 16:34:18 -07006134 spin_lock_init(&memcg->pcp_counter_lock);
6135 return memcg;
Dan Carpenterd2e61b82010-11-11 14:05:12 -08006136
6137out_free:
6138 if (size < PAGE_SIZE)
Hugh Dickinsd79154b2012-03-21 16:34:18 -07006139 kfree(memcg);
Dan Carpenterd2e61b82010-11-11 14:05:12 -08006140 else
Hugh Dickinsd79154b2012-03-21 16:34:18 -07006141 vfree(memcg);
Dan Carpenterd2e61b82010-11-11 14:05:12 -08006142 return NULL;
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07006143}
6144
KAMEZAWA Hiroyuki8c7c6e342009-01-07 18:08:00 -08006145/*
Glauber Costac8b2a362012-12-18 14:22:13 -08006146 * At destroying mem_cgroup, references from swap_cgroup can remain.
6147 * (scanning all at force_empty is too costly...)
6148 *
6149 * Instead of clearing all references at force_empty, we remember
6150 * the number of reference from swap_cgroup and free mem_cgroup when
6151 * it goes down to 0.
6152 *
6153 * Removal of cgroup itself succeeds regardless of refs from swap.
Hugh Dickins59927fb2012-03-15 15:17:07 -07006154 */
Glauber Costac8b2a362012-12-18 14:22:13 -08006155
6156static void __mem_cgroup_free(struct mem_cgroup *memcg)
Hugh Dickins59927fb2012-03-15 15:17:07 -07006157{
Glauber Costac8b2a362012-12-18 14:22:13 -08006158 int node;
Glauber Costa45cf7eb2013-02-22 16:34:49 -08006159 size_t size = memcg_size();
Hugh Dickins59927fb2012-03-15 15:17:07 -07006160
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -07006161 mem_cgroup_remove_from_trees(memcg);
Glauber Costac8b2a362012-12-18 14:22:13 -08006162 free_css_id(&mem_cgroup_subsys, &memcg->css);
6163
6164 for_each_node(node)
6165 free_mem_cgroup_per_zone_info(memcg, node);
6166
6167 free_percpu(memcg->stat);
6168
Glauber Costa3f134612012-05-29 15:07:11 -07006169 /*
6170 * We need to make sure that (at least for now), the jump label
6171 * destruction code runs outside of the cgroup lock. This is because
6172 * get_online_cpus(), which is called from the static_branch update,
6173 * can't be called inside the cgroup_lock. cpusets are the ones
6174 * enforcing this dependency, so if they ever change, we might as well.
6175 *
6176 * schedule_work() will guarantee this happens. Be careful if you need
6177 * to move this code around, and make sure it is outside
6178 * the cgroup_lock.
6179 */
Glauber Costaa8964b92012-12-18 14:22:09 -08006180 disarm_static_keys(memcg);
Glauber Costa3afe36b2012-05-29 15:07:10 -07006181 if (size < PAGE_SIZE)
6182 kfree(memcg);
6183 else
6184 vfree(memcg);
Hugh Dickins59927fb2012-03-15 15:17:07 -07006185}
Glauber Costa3afe36b2012-05-29 15:07:10 -07006186
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08006187/*
6188 * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
6189 */
Glauber Costae1aab162011-12-11 21:47:03 +00006190struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08006191{
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006192 if (!memcg->res.parent)
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08006193 return NULL;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006194 return mem_cgroup_from_res_counter(memcg->res.parent, res);
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08006195}
Glauber Costae1aab162011-12-11 21:47:03 +00006196EXPORT_SYMBOL(parent_mem_cgroup);
KAMEZAWA Hiroyuki33327942008-04-29 01:00:24 -07006197
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -07006198static void __init mem_cgroup_soft_limit_tree_init(void)
6199{
6200 struct mem_cgroup_tree_per_node *rtpn;
6201 struct mem_cgroup_tree_per_zone *rtpz;
6202 int tmp, node, zone;
6203
6204 for_each_node(node) {
6205 tmp = node;
6206 if (!node_state(node, N_NORMAL_MEMORY))
6207 tmp = -1;
6208 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
6209 BUG_ON(!rtpn);
6210
6211 soft_limit_tree.rb_tree_per_node[node] = rtpn;
6212
6213 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
6214 rtpz = &rtpn->rb_tree_per_zone[zone];
6215 rtpz->rb_root = RB_ROOT;
6216 spin_lock_init(&rtpz->lock);
6217 }
6218 }
6219}
6220
Li Zefan0eb253e2009-01-15 13:51:25 -08006221static struct cgroup_subsys_state * __ref
Tejun Heoeb954192013-08-08 20:11:23 -04006222mem_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006223{
Glauber Costad142e3e2013-02-22 16:34:52 -08006224 struct mem_cgroup *memcg;
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07006225 long error = -ENOMEM;
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006226 int node;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006227
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006228 memcg = mem_cgroup_alloc();
6229 if (!memcg)
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07006230 return ERR_PTR(error);
Pavel Emelianov78fb7462008-02-07 00:13:51 -08006231
Bob Liu3ed28fa2012-01-12 17:19:04 -08006232 for_each_node(node)
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006233 if (alloc_mem_cgroup_per_zone_info(memcg, node))
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006234 goto free_out;
Balbir Singhf64c3f52009-09-23 15:56:37 -07006235
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006236 /* root ? */
Tejun Heoeb954192013-08-08 20:11:23 -04006237 if (parent_css == NULL) {
Hillf Dantona41c58a2011-12-19 17:11:57 -08006238 root_mem_cgroup = memcg;
Glauber Costad142e3e2013-02-22 16:34:52 -08006239 res_counter_init(&memcg->res, NULL);
6240 res_counter_init(&memcg->memsw, NULL);
6241 res_counter_init(&memcg->kmem, NULL);
Balbir Singh18f59ea2009-01-07 18:08:07 -08006242 }
Balbir Singh28dbc4b2009-01-07 18:08:05 -08006243
Glauber Costad142e3e2013-02-22 16:34:52 -08006244 memcg->last_scanned_node = MAX_NUMNODES;
6245 INIT_LIST_HEAD(&memcg->oom_notify);
Glauber Costad142e3e2013-02-22 16:34:52 -08006246 memcg->move_charge_at_immigrate = 0;
6247 mutex_init(&memcg->thresholds_lock);
6248 spin_lock_init(&memcg->move_lock);
Anton Vorontsov70ddf632013-04-29 15:08:31 -07006249 vmpressure_init(&memcg->vmpressure);
Glauber Costad142e3e2013-02-22 16:34:52 -08006250
6251 return &memcg->css;
6252
6253free_out:
6254 __mem_cgroup_free(memcg);
6255 return ERR_PTR(error);
6256}
6257
6258static int
Tejun Heoeb954192013-08-08 20:11:23 -04006259mem_cgroup_css_online(struct cgroup_subsys_state *css)
Glauber Costad142e3e2013-02-22 16:34:52 -08006260{
Tejun Heoeb954192013-08-08 20:11:23 -04006261 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
6262 struct mem_cgroup *parent = mem_cgroup_from_css(css_parent(css));
Glauber Costad142e3e2013-02-22 16:34:52 -08006263 int error = 0;
6264
Tejun Heo63876982013-08-08 20:11:23 -04006265 if (!parent)
Glauber Costad142e3e2013-02-22 16:34:52 -08006266 return 0;
6267
Glauber Costa09998212013-02-22 16:34:55 -08006268 mutex_lock(&memcg_create_mutex);
Glauber Costad142e3e2013-02-22 16:34:52 -08006269
6270 memcg->use_hierarchy = parent->use_hierarchy;
6271 memcg->oom_kill_disable = parent->oom_kill_disable;
6272 memcg->swappiness = mem_cgroup_swappiness(parent);
6273
6274 if (parent->use_hierarchy) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006275 res_counter_init(&memcg->res, &parent->res);
6276 res_counter_init(&memcg->memsw, &parent->memsw);
Glauber Costa510fc4e2012-12-18 14:21:47 -08006277 res_counter_init(&memcg->kmem, &parent->kmem);
Glauber Costa55007d82012-12-18 14:22:38 -08006278
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08006279 /*
Li Zefan8d76a972013-07-08 16:00:36 -07006280 * No need to take a reference to the parent because cgroup
6281 * core guarantees its existence.
Daisuke Nishimura7bcc1bb2009-01-29 14:25:11 -08006282 */
Balbir Singh18f59ea2009-01-07 18:08:07 -08006283 } else {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006284 res_counter_init(&memcg->res, NULL);
6285 res_counter_init(&memcg->memsw, NULL);
Glauber Costa510fc4e2012-12-18 14:21:47 -08006286 res_counter_init(&memcg->kmem, NULL);
Tejun Heo8c7f6ed2012-09-13 12:20:58 -07006287 /*
6288 * Deeper hierachy with use_hierarchy == false doesn't make
6289 * much sense so let cgroup subsystem know about this
6290 * unfortunate state in our controller.
6291 */
Glauber Costad142e3e2013-02-22 16:34:52 -08006292 if (parent != root_mem_cgroup)
Tejun Heo8c7f6ed2012-09-13 12:20:58 -07006293 mem_cgroup_subsys.broken_hierarchy = true;
Balbir Singh18f59ea2009-01-07 18:08:07 -08006294 }
Glauber Costacbe128e32012-04-09 19:36:34 -03006295
6296 error = memcg_init_kmem(memcg, &mem_cgroup_subsys);
Glauber Costa09998212013-02-22 16:34:55 -08006297 mutex_unlock(&memcg_create_mutex);
Glauber Costad142e3e2013-02-22 16:34:52 -08006298 return error;
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006299}
6300
Michal Hocko5f578162013-04-29 15:07:17 -07006301/*
6302 * Announce all parents that a group from their hierarchy is gone.
6303 */
6304static void mem_cgroup_invalidate_reclaim_iterators(struct mem_cgroup *memcg)
6305{
6306 struct mem_cgroup *parent = memcg;
6307
6308 while ((parent = parent_mem_cgroup(parent)))
Johannes Weiner519ebea2013-07-03 15:04:51 -07006309 mem_cgroup_iter_invalidate(parent);
Michal Hocko5f578162013-04-29 15:07:17 -07006310
6311 /*
6312 * if the root memcg is not hierarchical we have to check it
6313 * explicitely.
6314 */
6315 if (!root_mem_cgroup->use_hierarchy)
Johannes Weiner519ebea2013-07-03 15:04:51 -07006316 mem_cgroup_iter_invalidate(root_mem_cgroup);
Michal Hocko5f578162013-04-29 15:07:17 -07006317}
6318
Tejun Heoeb954192013-08-08 20:11:23 -04006319static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08006320{
Tejun Heoeb954192013-08-08 20:11:23 -04006321 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
KAMEZAWA Hiroyukiec64f512009-04-02 16:57:26 -07006322
Li Zefan10d5ebf2013-07-08 16:00:33 -07006323 kmem_cgroup_css_offline(memcg);
6324
Michal Hocko5f578162013-04-29 15:07:17 -07006325 mem_cgroup_invalidate_reclaim_iterators(memcg);
Michal Hockoab5196c2012-10-26 13:37:32 +02006326 mem_cgroup_reparent_charges(memcg);
Glauber Costa1f458cb2012-12-18 14:22:50 -08006327 mem_cgroup_destroy_all_caches(memcg);
Michal Hocko33cb8762013-07-31 13:53:51 -07006328 vmpressure_cleanup(&memcg->vmpressure);
KAMEZAWA Hiroyukidf878fb2008-02-07 00:14:28 -08006329}
6330
Tejun Heoeb954192013-08-08 20:11:23 -04006331static void mem_cgroup_css_free(struct cgroup_subsys_state *css)
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006332{
Tejun Heoeb954192013-08-08 20:11:23 -04006333 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Daisuke Nishimurac268e992009-01-15 13:51:13 -08006334
Li Zefan10d5ebf2013-07-08 16:00:33 -07006335 memcg_destroy_kmem(memcg);
Li Zefan465939a2013-07-08 16:00:38 -07006336 __mem_cgroup_free(memcg);
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006337}
6338
Daisuke Nishimura02491442010-03-10 15:22:17 -08006339#ifdef CONFIG_MMU
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006340/* Handlers for move charge at task migration. */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006341#define PRECHARGE_COUNT_AT_ONCE 256
6342static int mem_cgroup_do_precharge(unsigned long count)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006343{
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006344 int ret = 0;
6345 int batch_count = PRECHARGE_COUNT_AT_ONCE;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006346 struct mem_cgroup *memcg = mc.to;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006347
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006348 if (mem_cgroup_is_root(memcg)) {
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006349 mc.precharge += count;
6350 /* we don't need css_get for root */
6351 return ret;
6352 }
6353 /* try to charge at once */
6354 if (count > 1) {
6355 struct res_counter *dummy;
6356 /*
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006357 * "memcg" cannot be under rmdir() because we've already checked
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006358 * by cgroup_lock_live_cgroup() that it is not removed and we
6359 * are still under the same cgroup_mutex. So we can postpone
6360 * css_get().
6361 */
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006362 if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy))
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006363 goto one_by_one;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006364 if (do_swap_account && res_counter_charge(&memcg->memsw,
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006365 PAGE_SIZE * count, &dummy)) {
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006366 res_counter_uncharge(&memcg->res, PAGE_SIZE * count);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006367 goto one_by_one;
6368 }
6369 mc.precharge += count;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006370 return ret;
6371 }
6372one_by_one:
6373 /* fall back to one by one charge */
6374 while (count--) {
6375 if (signal_pending(current)) {
6376 ret = -EINTR;
6377 break;
6378 }
6379 if (!batch_count--) {
6380 batch_count = PRECHARGE_COUNT_AT_ONCE;
6381 cond_resched();
6382 }
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006383 ret = __mem_cgroup_try_charge(NULL,
6384 GFP_KERNEL, 1, &memcg, false);
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08006385 if (ret)
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006386 /* mem_cgroup_clear_mc() will do uncharge later */
KAMEZAWA Hiroyuki38c5d722012-01-12 17:19:01 -08006387 return ret;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006388 mc.precharge++;
6389 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006390 return ret;
6391}
6392
6393/**
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006394 * get_mctgt_type - get target type of moving charge
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006395 * @vma: the vma the pte to be checked belongs
6396 * @addr: the address corresponding to the pte to be checked
6397 * @ptent: the pte to be checked
Daisuke Nishimura02491442010-03-10 15:22:17 -08006398 * @target: the pointer the target page or swap ent will be stored(can be NULL)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006399 *
6400 * Returns
6401 * 0(MC_TARGET_NONE): if the pte is not a target for move charge.
6402 * 1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
6403 * move charge. if @target is not NULL, the page is stored in target->page
6404 * with extra refcnt got(Callers should handle it).
Daisuke Nishimura02491442010-03-10 15:22:17 -08006405 * 2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
6406 * target for charge migration. if @target is not NULL, the entry is stored
6407 * in target->ent.
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006408 *
6409 * Called with pte lock held.
6410 */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006411union mc_target {
6412 struct page *page;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006413 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006414};
6415
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006416enum mc_target_type {
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006417 MC_TARGET_NONE = 0,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006418 MC_TARGET_PAGE,
Daisuke Nishimura02491442010-03-10 15:22:17 -08006419 MC_TARGET_SWAP,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006420};
6421
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006422static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
6423 unsigned long addr, pte_t ptent)
6424{
6425 struct page *page = vm_normal_page(vma, addr, ptent);
6426
6427 if (!page || !page_mapped(page))
6428 return NULL;
6429 if (PageAnon(page)) {
6430 /* we don't move shared anon */
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07006431 if (!move_anon())
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006432 return NULL;
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006433 } else if (!move_file())
6434 /* we ignore mapcount for file pages */
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006435 return NULL;
6436 if (!get_page_unless_zero(page))
6437 return NULL;
6438
6439 return page;
6440}
6441
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07006442#ifdef CONFIG_SWAP
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006443static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
6444 unsigned long addr, pte_t ptent, swp_entry_t *entry)
6445{
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006446 struct page *page = NULL;
6447 swp_entry_t ent = pte_to_swp_entry(ptent);
6448
6449 if (!move_anon() || non_swap_entry(ent))
6450 return NULL;
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07006451 /*
6452 * Because lookup_swap_cache() updates some statistics counter,
6453 * we call find_get_page() with swapper_space directly.
6454 */
Shaohua Li33806f02013-02-22 16:34:37 -08006455 page = find_get_page(swap_address_space(ent), ent.val);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006456 if (do_swap_account)
6457 entry->val = ent.val;
6458
6459 return page;
6460}
KAMEZAWA Hiroyuki4b913552012-05-29 15:06:51 -07006461#else
6462static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
6463 unsigned long addr, pte_t ptent, swp_entry_t *entry)
6464{
6465 return NULL;
6466}
6467#endif
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006468
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006469static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
6470 unsigned long addr, pte_t ptent, swp_entry_t *entry)
6471{
6472 struct page *page = NULL;
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006473 struct address_space *mapping;
6474 pgoff_t pgoff;
6475
6476 if (!vma->vm_file) /* anonymous vma */
6477 return NULL;
6478 if (!move_file())
6479 return NULL;
6480
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006481 mapping = vma->vm_file->f_mapping;
6482 if (pte_none(ptent))
6483 pgoff = linear_page_index(vma, addr);
6484 else /* pte_file(ptent) is true */
6485 pgoff = pte_to_pgoff(ptent);
6486
6487 /* page is moved even if it's not RSS of this task(page-faulted). */
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07006488 page = find_get_page(mapping, pgoff);
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006489
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07006490#ifdef CONFIG_SWAP
6491 /* shmem/tmpfs may report page out on swap: account for that too. */
6492 if (radix_tree_exceptional_entry(page)) {
6493 swp_entry_t swap = radix_to_swp_entry(page);
6494 if (do_swap_account)
6495 *entry = swap;
Shaohua Li33806f02013-02-22 16:34:37 -08006496 page = find_get_page(swap_address_space(swap), swap.val);
Hugh Dickinsaa3b1892011-08-03 16:21:24 -07006497 }
6498#endif
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006499 return page;
6500}
6501
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006502static enum mc_target_type get_mctgt_type(struct vm_area_struct *vma,
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006503 unsigned long addr, pte_t ptent, union mc_target *target)
6504{
Daisuke Nishimura02491442010-03-10 15:22:17 -08006505 struct page *page = NULL;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006506 struct page_cgroup *pc;
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006507 enum mc_target_type ret = MC_TARGET_NONE;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006508 swp_entry_t ent = { .val = 0 };
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006509
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006510 if (pte_present(ptent))
6511 page = mc_handle_present_pte(vma, addr, ptent);
6512 else if (is_swap_pte(ptent))
6513 page = mc_handle_swap_pte(vma, addr, ptent, &ent);
Daisuke Nishimura87946a72010-05-26 14:42:39 -07006514 else if (pte_none(ptent) || pte_file(ptent))
6515 page = mc_handle_file_pte(vma, addr, ptent, &ent);
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006516
6517 if (!page && !ent.val)
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006518 return ret;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006519 if (page) {
6520 pc = lookup_page_cgroup(page);
6521 /*
6522 * Do only loose check w/o page_cgroup lock.
6523 * mem_cgroup_move_account() checks the pc is valid or not under
6524 * the lock.
6525 */
6526 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
6527 ret = MC_TARGET_PAGE;
6528 if (target)
6529 target->page = page;
6530 }
6531 if (!ret || !target)
6532 put_page(page);
6533 }
Daisuke Nishimura90254a62010-05-26 14:42:38 -07006534 /* There is a swap entry and a page doesn't exist or isn't charged */
6535 if (ent.val && !ret &&
Bob Liu9fb4b7c2012-01-12 17:18:48 -08006536 css_id(&mc.from->css) == lookup_swap_cgroup_id(ent)) {
KAMEZAWA Hiroyuki7f0f1542010-05-11 14:06:58 -07006537 ret = MC_TARGET_SWAP;
6538 if (target)
6539 target->ent = ent;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006540 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006541 return ret;
6542}
6543
Naoya Horiguchi12724852012-03-21 16:34:28 -07006544#ifdef CONFIG_TRANSPARENT_HUGEPAGE
6545/*
6546 * We don't consider swapping or file mapped pages because THP does not
6547 * support them for now.
6548 * Caller should make sure that pmd_trans_huge(pmd) is true.
6549 */
6550static enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
6551 unsigned long addr, pmd_t pmd, union mc_target *target)
6552{
6553 struct page *page = NULL;
6554 struct page_cgroup *pc;
6555 enum mc_target_type ret = MC_TARGET_NONE;
6556
6557 page = pmd_page(pmd);
6558 VM_BUG_ON(!page || !PageHead(page));
6559 if (!move_anon())
6560 return ret;
6561 pc = lookup_page_cgroup(page);
6562 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
6563 ret = MC_TARGET_PAGE;
6564 if (target) {
6565 get_page(page);
6566 target->page = page;
6567 }
6568 }
6569 return ret;
6570}
6571#else
6572static inline enum mc_target_type get_mctgt_type_thp(struct vm_area_struct *vma,
6573 unsigned long addr, pmd_t pmd, union mc_target *target)
6574{
6575 return MC_TARGET_NONE;
6576}
6577#endif
6578
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006579static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
6580 unsigned long addr, unsigned long end,
6581 struct mm_walk *walk)
6582{
6583 struct vm_area_struct *vma = walk->private;
6584 pte_t *pte;
6585 spinlock_t *ptl;
6586
Naoya Horiguchi12724852012-03-21 16:34:28 -07006587 if (pmd_trans_huge_lock(pmd, vma) == 1) {
6588 if (get_mctgt_type_thp(vma, addr, *pmd, NULL) == MC_TARGET_PAGE)
6589 mc.precharge += HPAGE_PMD_NR;
6590 spin_unlock(&vma->vm_mm->page_table_lock);
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07006591 return 0;
Naoya Horiguchi12724852012-03-21 16:34:28 -07006592 }
Dave Hansen03319322011-03-22 16:32:56 -07006593
Andrea Arcangeli45f83ce2012-03-28 14:42:40 -07006594 if (pmd_trans_unstable(pmd))
6595 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006596 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
6597 for (; addr != end; pte++, addr += PAGE_SIZE)
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006598 if (get_mctgt_type(vma, addr, *pte, NULL))
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006599 mc.precharge++; /* increment precharge temporarily */
6600 pte_unmap_unlock(pte - 1, ptl);
6601 cond_resched();
6602
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006603 return 0;
6604}
6605
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006606static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
6607{
6608 unsigned long precharge;
6609 struct vm_area_struct *vma;
6610
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006611 down_read(&mm->mmap_sem);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006612 for (vma = mm->mmap; vma; vma = vma->vm_next) {
6613 struct mm_walk mem_cgroup_count_precharge_walk = {
6614 .pmd_entry = mem_cgroup_count_precharge_pte_range,
6615 .mm = mm,
6616 .private = vma,
6617 };
6618 if (is_vm_hugetlb_page(vma))
6619 continue;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006620 walk_page_range(vma->vm_start, vma->vm_end,
6621 &mem_cgroup_count_precharge_walk);
6622 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006623 up_read(&mm->mmap_sem);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006624
6625 precharge = mc.precharge;
6626 mc.precharge = 0;
6627
6628 return precharge;
6629}
6630
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006631static int mem_cgroup_precharge_mc(struct mm_struct *mm)
6632{
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006633 unsigned long precharge = mem_cgroup_count_precharge(mm);
6634
6635 VM_BUG_ON(mc.moving_task);
6636 mc.moving_task = current;
6637 return mem_cgroup_do_precharge(precharge);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006638}
6639
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006640/* cancels all extra charges on mc.from and mc.to, and wakes up all waiters. */
6641static void __mem_cgroup_clear_mc(void)
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006642{
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07006643 struct mem_cgroup *from = mc.from;
6644 struct mem_cgroup *to = mc.to;
Li Zefan40503772013-07-08 16:00:34 -07006645 int i;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07006646
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006647 /* we must uncharge all the leftover precharges from mc.to */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006648 if (mc.precharge) {
6649 __mem_cgroup_cancel_charge(mc.to, mc.precharge);
6650 mc.precharge = 0;
6651 }
6652 /*
6653 * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
6654 * we must uncharge here.
6655 */
6656 if (mc.moved_charge) {
6657 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
6658 mc.moved_charge = 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006659 }
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006660 /* we must fixup refcnts and charges */
6661 if (mc.moved_swap) {
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006662 /* uncharge swap account from the old cgroup */
6663 if (!mem_cgroup_is_root(mc.from))
6664 res_counter_uncharge(&mc.from->memsw,
6665 PAGE_SIZE * mc.moved_swap);
Li Zefan40503772013-07-08 16:00:34 -07006666
6667 for (i = 0; i < mc.moved_swap; i++)
6668 css_put(&mc.from->css);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006669
6670 if (!mem_cgroup_is_root(mc.to)) {
6671 /*
6672 * we charged both to->res and to->memsw, so we should
6673 * uncharge to->res.
6674 */
6675 res_counter_uncharge(&mc.to->res,
6676 PAGE_SIZE * mc.moved_swap);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006677 }
Li Zefan40503772013-07-08 16:00:34 -07006678 /* we've already done css_get(mc.to) */
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006679 mc.moved_swap = 0;
6680 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006681 memcg_oom_recover(from);
6682 memcg_oom_recover(to);
6683 wake_up_all(&mc.waitq);
6684}
6685
6686static void mem_cgroup_clear_mc(void)
6687{
6688 struct mem_cgroup *from = mc.from;
6689
6690 /*
6691 * we must clear moving_task before waking up waiters at the end of
6692 * task migration.
6693 */
6694 mc.moving_task = NULL;
6695 __mem_cgroup_clear_mc();
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07006696 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006697 mc.from = NULL;
6698 mc.to = NULL;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07006699 spin_unlock(&mc.lock);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07006700 mem_cgroup_end_move(from);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006701}
6702
Tejun Heoeb954192013-08-08 20:11:23 -04006703static int mem_cgroup_can_attach(struct cgroup_subsys_state *css,
Li Zefan761b3ef2012-01-31 13:47:36 +08006704 struct cgroup_taskset *tset)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006705{
Tejun Heo2f7ee562011-12-12 18:12:21 -08006706 struct task_struct *p = cgroup_taskset_first(tset);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006707 int ret = 0;
Tejun Heoeb954192013-08-08 20:11:23 -04006708 struct mem_cgroup *memcg = mem_cgroup_from_css(css);
Glauber Costaee5e8472013-02-22 16:34:50 -08006709 unsigned long move_charge_at_immigrate;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006710
Glauber Costaee5e8472013-02-22 16:34:50 -08006711 /*
6712 * We are now commited to this value whatever it is. Changes in this
6713 * tunable will only affect upcoming migrations, not the current one.
6714 * So we need to save it, and keep it going.
6715 */
6716 move_charge_at_immigrate = memcg->move_charge_at_immigrate;
6717 if (move_charge_at_immigrate) {
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006718 struct mm_struct *mm;
6719 struct mem_cgroup *from = mem_cgroup_from_task(p);
6720
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006721 VM_BUG_ON(from == memcg);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006722
6723 mm = get_task_mm(p);
6724 if (!mm)
6725 return 0;
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006726 /* We move charges only when we move a owner of the mm */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006727 if (mm->owner == p) {
6728 VM_BUG_ON(mc.from);
6729 VM_BUG_ON(mc.to);
6730 VM_BUG_ON(mc.precharge);
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006731 VM_BUG_ON(mc.moved_charge);
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006732 VM_BUG_ON(mc.moved_swap);
KAMEZAWA Hiroyuki32047e22010-10-27 15:33:40 -07006733 mem_cgroup_start_move(from);
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07006734 spin_lock(&mc.lock);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006735 mc.from = from;
Raghavendra K Tc0ff4b82011-11-02 13:38:15 -07006736 mc.to = memcg;
Glauber Costaee5e8472013-02-22 16:34:50 -08006737 mc.immigrate_flags = move_charge_at_immigrate;
KAMEZAWA Hiroyuki2bd9bb22010-08-10 18:02:58 -07006738 spin_unlock(&mc.lock);
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006739 /* We set mc.moving_task later */
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006740
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006741 ret = mem_cgroup_precharge_mc(mm);
6742 if (ret)
6743 mem_cgroup_clear_mc();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006744 }
6745 mmput(mm);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006746 }
6747 return ret;
6748}
6749
Tejun Heoeb954192013-08-08 20:11:23 -04006750static void mem_cgroup_cancel_attach(struct cgroup_subsys_state *css,
Li Zefan761b3ef2012-01-31 13:47:36 +08006751 struct cgroup_taskset *tset)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006752{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006753 mem_cgroup_clear_mc();
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006754}
6755
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006756static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
6757 unsigned long addr, unsigned long end,
6758 struct mm_walk *walk)
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006759{
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006760 int ret = 0;
6761 struct vm_area_struct *vma = walk->private;
6762 pte_t *pte;
6763 spinlock_t *ptl;
Naoya Horiguchi12724852012-03-21 16:34:28 -07006764 enum mc_target_type target_type;
6765 union mc_target target;
6766 struct page *page;
6767 struct page_cgroup *pc;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006768
Naoya Horiguchi12724852012-03-21 16:34:28 -07006769 /*
6770 * We don't take compound_lock() here but no race with splitting thp
6771 * happens because:
6772 * - if pmd_trans_huge_lock() returns 1, the relevant thp is not
6773 * under splitting, which means there's no concurrent thp split,
6774 * - if another thread runs into split_huge_page() just after we
6775 * entered this if-block, the thread must wait for page table lock
6776 * to be unlocked in __split_huge_page_splitting(), where the main
6777 * part of thp split is not executed yet.
6778 */
6779 if (pmd_trans_huge_lock(pmd, vma) == 1) {
Hugh Dickins62ade862012-05-18 11:28:34 -07006780 if (mc.precharge < HPAGE_PMD_NR) {
Naoya Horiguchi12724852012-03-21 16:34:28 -07006781 spin_unlock(&vma->vm_mm->page_table_lock);
6782 return 0;
6783 }
6784 target_type = get_mctgt_type_thp(vma, addr, *pmd, &target);
6785 if (target_type == MC_TARGET_PAGE) {
6786 page = target.page;
6787 if (!isolate_lru_page(page)) {
6788 pc = lookup_page_cgroup(page);
6789 if (!mem_cgroup_move_account(page, HPAGE_PMD_NR,
KAMEZAWA Hiroyuki2f3479b2012-05-29 15:07:04 -07006790 pc, mc.from, mc.to)) {
Naoya Horiguchi12724852012-03-21 16:34:28 -07006791 mc.precharge -= HPAGE_PMD_NR;
6792 mc.moved_charge += HPAGE_PMD_NR;
6793 }
6794 putback_lru_page(page);
6795 }
6796 put_page(page);
6797 }
6798 spin_unlock(&vma->vm_mm->page_table_lock);
Andrea Arcangeli1a5a9902012-03-21 16:33:42 -07006799 return 0;
Naoya Horiguchi12724852012-03-21 16:34:28 -07006800 }
6801
Andrea Arcangeli45f83ce2012-03-28 14:42:40 -07006802 if (pmd_trans_unstable(pmd))
6803 return 0;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006804retry:
6805 pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
6806 for (; addr != end; addr += PAGE_SIZE) {
6807 pte_t ptent = *(pte++);
Daisuke Nishimura02491442010-03-10 15:22:17 -08006808 swp_entry_t ent;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006809
6810 if (!mc.precharge)
6811 break;
6812
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006813 switch (get_mctgt_type(vma, addr, ptent, &target)) {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006814 case MC_TARGET_PAGE:
6815 page = target.page;
6816 if (isolate_lru_page(page))
6817 goto put;
6818 pc = lookup_page_cgroup(page);
Johannes Weiner7ec99d62011-03-23 16:42:36 -07006819 if (!mem_cgroup_move_account(page, 1, pc,
KAMEZAWA Hiroyuki2f3479b2012-05-29 15:07:04 -07006820 mc.from, mc.to)) {
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006821 mc.precharge--;
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006822 /* we uncharge from mc.from later. */
6823 mc.moved_charge++;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006824 }
6825 putback_lru_page(page);
Naoya Horiguchi8d32ff82012-03-21 16:34:27 -07006826put: /* get_mctgt_type() gets the page */
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006827 put_page(page);
6828 break;
Daisuke Nishimura02491442010-03-10 15:22:17 -08006829 case MC_TARGET_SWAP:
6830 ent = target.ent;
Hugh Dickinse91cbb42012-05-29 15:06:51 -07006831 if (!mem_cgroup_move_swap_account(ent, mc.from, mc.to)) {
Daisuke Nishimura02491442010-03-10 15:22:17 -08006832 mc.precharge--;
Daisuke Nishimura483c30b2010-03-10 15:22:18 -08006833 /* we fixup refcnts and charges later. */
6834 mc.moved_swap++;
6835 }
Daisuke Nishimura02491442010-03-10 15:22:17 -08006836 break;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006837 default:
6838 break;
6839 }
6840 }
6841 pte_unmap_unlock(pte - 1, ptl);
6842 cond_resched();
6843
6844 if (addr != end) {
6845 /*
6846 * We have consumed all precharges we got in can_attach().
6847 * We try charge one by one, but don't do any additional
6848 * charges to mc.to if we have failed in charge once in attach()
6849 * phase.
6850 */
Daisuke Nishimura854ffa82010-03-10 15:22:15 -08006851 ret = mem_cgroup_do_precharge(1);
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006852 if (!ret)
6853 goto retry;
6854 }
6855
6856 return ret;
6857}
6858
6859static void mem_cgroup_move_charge(struct mm_struct *mm)
6860{
6861 struct vm_area_struct *vma;
6862
6863 lru_add_drain_all();
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006864retry:
6865 if (unlikely(!down_read_trylock(&mm->mmap_sem))) {
6866 /*
6867 * Someone who are holding the mmap_sem might be waiting in
6868 * waitq. So we cancel all extra charges, wake up all waiters,
6869 * and retry. Because we cancel precharges, we might not be able
6870 * to move enough charges, but moving charge is a best-effort
6871 * feature anyway, so it wouldn't be a big problem.
6872 */
6873 __mem_cgroup_clear_mc();
6874 cond_resched();
6875 goto retry;
6876 }
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006877 for (vma = mm->mmap; vma; vma = vma->vm_next) {
6878 int ret;
6879 struct mm_walk mem_cgroup_move_charge_walk = {
6880 .pmd_entry = mem_cgroup_move_charge_pte_range,
6881 .mm = mm,
6882 .private = vma,
6883 };
6884 if (is_vm_hugetlb_page(vma))
6885 continue;
Daisuke Nishimura4ffef5f2010-03-10 15:22:14 -08006886 ret = walk_page_range(vma->vm_start, vma->vm_end,
6887 &mem_cgroup_move_charge_walk);
6888 if (ret)
6889 /*
6890 * means we have consumed all precharges and failed in
6891 * doing additional charge. Just abandon here.
6892 */
6893 break;
6894 }
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006895 up_read(&mm->mmap_sem);
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006896}
6897
Tejun Heoeb954192013-08-08 20:11:23 -04006898static void mem_cgroup_move_task(struct cgroup_subsys_state *css,
Li Zefan761b3ef2012-01-31 13:47:36 +08006899 struct cgroup_taskset *tset)
Balbir Singh67e465a2008-02-07 00:13:54 -08006900{
Tejun Heo2f7ee562011-12-12 18:12:21 -08006901 struct task_struct *p = cgroup_taskset_first(tset);
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07006902 struct mm_struct *mm = get_task_mm(p);
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006903
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006904 if (mm) {
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07006905 if (mc.to)
6906 mem_cgroup_move_charge(mm);
Daisuke Nishimuradfe076b2011-01-13 15:47:41 -08006907 mmput(mm);
6908 }
KOSAKI Motohiroa4336582011-06-15 15:08:13 -07006909 if (mc.to)
6910 mem_cgroup_clear_mc();
Balbir Singh67e465a2008-02-07 00:13:54 -08006911}
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006912#else /* !CONFIG_MMU */
Tejun Heoeb954192013-08-08 20:11:23 -04006913static int mem_cgroup_can_attach(struct cgroup_subsys_state *css,
Li Zefan761b3ef2012-01-31 13:47:36 +08006914 struct cgroup_taskset *tset)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006915{
6916 return 0;
6917}
Tejun Heoeb954192013-08-08 20:11:23 -04006918static void mem_cgroup_cancel_attach(struct cgroup_subsys_state *css,
Li Zefan761b3ef2012-01-31 13:47:36 +08006919 struct cgroup_taskset *tset)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006920{
6921}
Tejun Heoeb954192013-08-08 20:11:23 -04006922static void mem_cgroup_move_task(struct cgroup_subsys_state *css,
Li Zefan761b3ef2012-01-31 13:47:36 +08006923 struct cgroup_taskset *tset)
Daisuke Nishimura5cfb80a2010-03-23 13:35:11 -07006924{
6925}
6926#endif
Balbir Singh67e465a2008-02-07 00:13:54 -08006927
Tejun Heof00baae2013-04-15 13:41:15 -07006928/*
6929 * Cgroup retains root cgroups across [un]mount cycles making it necessary
6930 * to verify sane_behavior flag on each mount attempt.
6931 */
Tejun Heoeb954192013-08-08 20:11:23 -04006932static void mem_cgroup_bind(struct cgroup_subsys_state *root_css)
Tejun Heof00baae2013-04-15 13:41:15 -07006933{
6934 /*
6935 * use_hierarchy is forced with sane_behavior. cgroup core
6936 * guarantees that @root doesn't have any children, so turning it
6937 * on for the root memcg is enough.
6938 */
Tejun Heoeb954192013-08-08 20:11:23 -04006939 if (cgroup_sane_behavior(root_css->cgroup))
6940 mem_cgroup_from_css(root_css)->use_hierarchy = true;
Tejun Heof00baae2013-04-15 13:41:15 -07006941}
6942
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006943struct cgroup_subsys mem_cgroup_subsys = {
6944 .name = "memory",
6945 .subsys_id = mem_cgroup_subsys_id,
Tejun Heo92fb9742012-11-19 08:13:38 -08006946 .css_alloc = mem_cgroup_css_alloc,
Glauber Costad142e3e2013-02-22 16:34:52 -08006947 .css_online = mem_cgroup_css_online,
Tejun Heo92fb9742012-11-19 08:13:38 -08006948 .css_offline = mem_cgroup_css_offline,
6949 .css_free = mem_cgroup_css_free,
Daisuke Nishimura7dc74be2010-03-10 15:22:13 -08006950 .can_attach = mem_cgroup_can_attach,
6951 .cancel_attach = mem_cgroup_cancel_attach,
Balbir Singh67e465a2008-02-07 00:13:54 -08006952 .attach = mem_cgroup_move_task,
Tejun Heof00baae2013-04-15 13:41:15 -07006953 .bind = mem_cgroup_bind,
Tejun Heo6bc10342012-04-01 12:09:55 -07006954 .base_cftypes = mem_cgroup_files,
KAMEZAWA Hiroyuki6d12e2d2008-02-07 00:14:31 -08006955 .early_init = 0,
KAMEZAWA Hiroyuki04046e12009-04-02 16:57:33 -07006956 .use_id = 1,
Balbir Singh8cdea7c2008-02-07 00:13:50 -08006957};
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006958
Andrew Mortonc255a452012-07-31 16:43:02 -07006959#ifdef CONFIG_MEMCG_SWAP
Michal Hockoa42c3902010-11-24 12:57:08 -08006960static int __init enable_swap_account(char *s)
6961{
Michal Hockoa2c89902011-05-24 17:12:50 -07006962 if (!strcmp(s, "1"))
Michal Hockoa42c3902010-11-24 12:57:08 -08006963 really_do_swap_account = 1;
Michal Hockoa2c89902011-05-24 17:12:50 -07006964 else if (!strcmp(s, "0"))
Michal Hockoa42c3902010-11-24 12:57:08 -08006965 really_do_swap_account = 0;
6966 return 1;
6967}
Michal Hockoa2c89902011-05-24 17:12:50 -07006968__setup("swapaccount=", enable_swap_account);
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006969
Michal Hocko2d110852013-02-22 16:34:43 -08006970static void __init memsw_file_init(void)
6971{
Michal Hocko6acc8b02013-02-22 16:34:45 -08006972 WARN_ON(cgroup_add_cftypes(&mem_cgroup_subsys, memsw_cgroup_files));
Michal Hocko2d110852013-02-22 16:34:43 -08006973}
Michal Hocko6acc8b02013-02-22 16:34:45 -08006974
6975static void __init enable_swap_cgroup(void)
6976{
6977 if (!mem_cgroup_disabled() && really_do_swap_account) {
6978 do_swap_account = 1;
6979 memsw_file_init();
6980 }
6981}
6982
Michal Hocko2d110852013-02-22 16:34:43 -08006983#else
Michal Hocko6acc8b02013-02-22 16:34:45 -08006984static void __init enable_swap_cgroup(void)
Michal Hocko2d110852013-02-22 16:34:43 -08006985{
6986}
KAMEZAWA Hiroyukic0777192009-01-07 18:07:57 -08006987#endif
Michal Hocko2d110852013-02-22 16:34:43 -08006988
6989/*
Michal Hocko10813122013-02-22 16:35:41 -08006990 * subsys_initcall() for memory controller.
6991 *
6992 * Some parts like hotcpu_notifier() have to be initialized from this context
6993 * because of lock dependencies (cgroup_lock -> cpu hotplug) but basically
6994 * everything that doesn't depend on a specific mem_cgroup structure should
6995 * be initialized from here.
Michal Hocko2d110852013-02-22 16:34:43 -08006996 */
6997static int __init mem_cgroup_init(void)
6998{
6999 hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
Michal Hocko6acc8b02013-02-22 16:34:45 -08007000 enable_swap_cgroup();
Andrew Mortonbb4cc1a2013-09-24 15:27:40 -07007001 mem_cgroup_soft_limit_tree_init();
Michal Hockoe4777492013-02-22 16:35:40 -08007002 memcg_stock_init();
Michal Hocko2d110852013-02-22 16:34:43 -08007003 return 0;
7004}
7005subsys_initcall(mem_cgroup_init);